From 853c92f35f3b4d3df841958ba33568f690f86c5d Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Wed, 18 Nov 2026 12:52:47 +0800 Subject: [PATCH 1/8] Add offline candidate trees and verification completion menus Separate editable lineage from candidate-specific verification evidence. Retain scoped attempts and dependency epochs, select complete development candidates, and enumerate bounded completion plans with shared preparation and cost caps. Cover failed-parent repair, stale evidence, deterministic selection and planner feasibility with CPU controls; document the shared execution integration boundary. Agent-assisted-by: Codex (implementation, tests and documentation) Signed-off-by: amy-why-3459 --- docs/architecture.md | 5 + docs/operations.md | 1 + docs/speculative-search.md | 140 ++++++++ src/inferrsi/harness/search_tree.py | 361 +++++++++++++++++++++ src/inferrsi/harness/verification_plans.py | 145 +++++++++ tests/fixtures/speculative_search.py | 67 ++++ tests/test_speculative_search.py | 263 +++++++++++++++ tests/test_verification_plans.py | 151 +++++++++ 8 files changed, 1133 insertions(+) create mode 100644 docs/speculative-search.md create mode 100644 src/inferrsi/harness/search_tree.py create mode 100644 src/inferrsi/harness/verification_plans.py create mode 100644 tests/fixtures/speculative_search.py create mode 100644 tests/test_speculative_search.py create mode 100644 tests/test_verification_plans.py diff --git a/docs/architecture.md b/docs/architecture.md index 4a14f040..e2f05e8e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -28,6 +28,11 @@ does not import Benchmark. Benchmark supplies task definitions and specialized rules, while Harness manages their execution and the complete optimization lifecycle. Neither infers quality criteria from a natural-language goal. Ordinary projects default to the current Agent conversation. +The [offline speculative-search primitives](speculative-search.md) add immutable +candidate trees, scoped evidence transitions and bounded completion menus in +Harness. They expose pure control operations for future VATS assembly; the +existing session controller continues to own execution. Production adapter +integration, learned role selection and GVO remain separate delivery work. The [native host handoff](native-agent-sessions.md) lets the initiating Agent resolve the task request before any campaign exists, use its host's new-session tool and retain the native identity and returned artifacts. The child owns the diff --git a/docs/operations.md b/docs/operations.md index 806b2602..176e0c8c 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -65,6 +65,7 @@ Effect classes used below: | Qualify an artifact verifier host | [Manual reviewer qualification](skill-eval-reviewer-qualification.md), `python -m tools.skill_eval_qualify` | Trusted-control execution / report-generating | Manual main-only workflow; fixed software controls, no model or external artifact input; unavailable isolation fails rather than skips | | Replay a cloud-generated parser pair | [Isolated cloud replay](skill-eval-cloud-replay.md), `python -m tools.skill_eval_cloud_replay` | Untrusted-artifact execution / report-generating | Manual pinned data ingress; same-host controls before bubblewrap-only replay; no models or credentials | | Validate one harness-improvement step | [Offline improvement](harness-improvement.md), Python `tools.skill_eval_improve.prepare` / `run_offline` | Snapshot-writing / offline experiment-executing | Supplied declarative INV/DOM edits, shared total limits, rerun and one holdout gate; trusted fixed-output clients only; no real-model or Store execution | +| Build an offline candidate tree and completion menu | Python `harness.search_tree` / `harness.verification_plans.completion_menu`; [control contract](speculative-search.md) | Read-only / in-memory transitions | Immutable candidates, scoped caller-supplied receipts and bounded exact plans; execution, durable state, accounting, learned roles and final acceptance require separate adapters | | Plan and report skill PR evaluations | [Skill PR integration](skill-eval-pr.md), `python -m tools.skill_eval_pr` | Read-only planning / authorized experiment-executing / PR-publishing | Trusted mapping, exact operator approval, consumed execution identity and credential-separated publication from retained evidence; no automatic paid trials | | Recover goal progress and the next proposal | [Offline goal checkpoints](goal-checkpoints.md), `python -m tools.goal_checkpoint` | Read-only | Explicit scope/criteria, pinned owner decisions and existing comparison receipts; a disposable view, never execution or acceptance authority | | Summarize campaign gains | `tools/campaign_summary_chart.py`; `report`/`run` figure | Figure-generating | Renders recorded values into one gain figure plus a matching table; validates the specification and computes nothing new | diff --git a/docs/speculative-search.md b/docs/speculative-search.md new file mode 100644 index 00000000..96d6e38a --- /dev/null +++ b/docs/speculative-search.md @@ -0,0 +1,140 @@ +# Offline candidate trees and verification plans + +This is the first implementation slice of +[the SpecRSI proposal](https://github.com/InferMatrixLab/InferRSI/issues/363). +`harness/search_tree.py` provides immutable candidate and evidence transitions; +`harness/verification_plans.py` constructs bounded completion menus. Both are +pure Python control primitives with no execution, storage or model access. +The existing project and benchmark session commands retain their current behavior. + +The design follows [SpecRSI v2, Section 3](https://arxiv.org/html/2610.16491v2#S3): +editing an incomplete or unsuccessful parent is allowed, while adopting a version +requires complete admissible evidence for that version. Completion bundles keep +complementary checks together. This slice supplies building blocks for the inner +search; a learned Scorer/Policy, automatic Drafter and GVO updates remain future +integration work. + +## State and evidence + +`Contract` freezes the search identity, verification context, required checks, +objective, direction, improver snapshot and candidate/depth/check-attempt caps. +The context identifies the plan, validator, environment, workload, original +baseline and quality reference. Checks may encode separately required repetitions. +The current selector supports a scalar objective plus hard PASS requirements; +the Skill lane's quality-first lexicographic selector needs a separate adapter. + +`initialize` requires one complete, passing receipt per root check. Root receipts +count against the attempt cap. Their acquisition/reuse cost must already be in +the caller's authoritative accounting. A passing objective check carries a finite +value; other checks and rejected/inconclusive outcomes cannot supply a value. + +`add_candidate` records an immutable artifact digest and edit parent, with the +frozen producer snapshot, producer revision, context/history digests and actual +generation dependencies. Callers retain and verify artifacts with the existing +`harness.candidates.retain` / `identity` helpers before admission. This core does +not authenticate digests or decide whether an edit was permitted. Parent quality +failures do not invalidate the edit lineage. Real generation-source dependencies +are inherited and must still be current; a child cannot drop them to escape an +invalidation. Rebase or composition is a new candidate with its own evidence. +This offline core predeclares dependency keys at initialization; introducing new +source authorities dynamically requires later controller integration. + +`register_attempt` binds a caller-owned attempt/execution key, reservation and +lease to one candidate, check and complete verification context. Registering an +attempt grants no execution permission. Unknown work on the same check cannot be +replaced, even after dependency invalidation. Valid outcomes cannot be resampled +until they pass; an INCONCLUSIVE outcome may be retried with a fresh identity +under the caller's frozen retry rules and a new reservation. + +`record_receipt` requires the entire registered binding. Identical delivery is +idempotent and conflicting delivery is rejected. History retains late results; +`evidence` admits only the latest attempt with current dependency epochs and +valid candidate provenance. FAIL is an observed outcome, whereas INCONCLUSIVE +leaves a check missing. Identical candidate bytes do not transfer evidence. + +`accept` selects only fully passing candidates, preferring the root on an +objective tie and otherwise using lexicographic candidate ID order. The result is +a **development selection**. Independent final/holdout acceptance and production +promotion remain caller-owned. `verified_gain` measures this selection relative +to the root. Invalidation recomputes eligibility; invalid root evidence blocks +acceptance and further drafting/check registration, while late archival remains +available. + +## Completion planning + +`completion_menu(state, jobs, available_budget=..., available_jobs=..., +max_targets=..., max_jobs=...)` takes a supplied integer-cost budget view and +returns immutable `Plan` values. The caller has already accounted for planning, +generation, final reserve, in-flight obligations and unknown spend when deriving +that view. Costs must be known positive conservative bounds in the same unit. + +Each `Job` names either a candidate/check pair or a shared preparation step. +Prerequisites form an acyclic graph. The solver takes each candidate's complete +missing-check set, closes its prerequisites, and enumerates unions of at most +`max_targets` candidate seeds. Every feasible singleton completion remains in +the menu. Shared jobs are charged once per plan, conflicts exclude combinations, +and available job slots and remaining check attempts bound each proposal. +`max_jobs` bounds the input graph; enumeration is intended for small offline +controls, not an unbounded production optimizer. No Scorer prediction is used +as evidence or as a substitute for a completion seed. + +Plans bind the search ID and state revision and carry a deterministic topological +job order. The caller retains the full job definitions and costs with its frozen +plan; job IDs alone do not authenticate them. The adapter rebuilds job +relations from current observations at each synchronous decision boundary; +already observed checks are not silently scheduled again. An in-flight attempt, +including an invalidated one, blocks replanning until its execution is settled. +An empty menu is a valid outcome under a tight budget; the root remains available +when its evidence is valid. Feasibility says nothing about whether checks pass. + +## Shared execution and persistence boundary + +These modules introduce no second runner, budget ledger, holdout tracker or +storage backend. A future application adapter must: + +1. Read current authoritative state and the frozen contract, settle the previous + work block, and supply a conservative available-budget view. +2. Generate and retain permitted edits, or select an affordable check plan using + the frozen roles. Account for generation, planning and failed work. +3. Use shared execution controls for atomic claim/reservation, absolute deadline, + resource isolation and actual execution. Bind their identities to the tree. +4. Authenticate and assess receipts, apply a transition against the expected + search revision, and persist it using the authoritative storage CAS/lock. + A pure function's revision check is **not a durable or cross-worker CAS**. + On conflict, reread and recompute; an ambiguous claim never grants a retry. +5. Persist real dependency invalidation before cancellation. Reconcile costs and + retain leases until process cleanup is confirmed. Receipt archival itself + neither charges again nor releases a resource. +6. Freeze the selected version for independent final acceptance, with holdout + access and interruptions tracked separately from development feedback. + +There is no production adapter in this slice. In particular, local immutable +values and revision checks do not establish crash recovery, external exactly-once, +multi-worker resource isolation, budget transactions or holdout non-reuse. + +## Verification and remaining delivery + +Run the CPU controls with: + +```bash +python -m pytest tests/test_speculative_search.py tests/test_verification_plans.py +``` + +Tests cover retained file/directory artifacts for synthetic Engine/Skill shapes, +failed-parent repair, incomplete ancestors, failure versus missing evidence, +dependency/result order, identical bytes with separate attempts, stale revision +rejection, immutable role snapshots, deterministic selection, invalid roots and +bounded search. The planner is compared with an independent exhaustive job-subset +oracle across budgets and target limits, including shared preparation and conflicts. + +| Issue acceptance area | This slice | Remaining evidence | +| --- | --- | --- | +| AC-01/02 | Frozen scalar contract and offline candidate tree | Production support matrix and role/execution adapters | +| AC-03/05 | Scoped evidence, epoch invalidation, development selection and revision guards | Authoritative durable state integration and restart/race tests | +| AC-04 | Bounded exact completion menu and exhaustive controls | Scorer refinement, Policy selection and scalable solver integration | +| AC-06 | Frozen snapshot identity only | GVO data collection, objectives, parameter updates and new searches | +| AC-07/08 | References to caller-owned execution identities | Actual budget/resource transactions, cancellation and independent final/holdout | +| AC-09/10 | Synthetic control fixtures | Both real adapters, model generation, training, matched experiments and lifecycle costs | + +These tests do not establish model quality, engine speedup or SpecRSI efficiency. +The parent issue remains open for its full dual-loop delivery. diff --git a/src/inferrsi/harness/search_tree.py b/src/inferrsi/harness/search_tree.py new file mode 100644 index 00000000..d6623835 --- /dev/null +++ b/src/inferrsi/harness/search_tree.py @@ -0,0 +1,361 @@ +"""Pure VATS candidate/evidence transitions; execution and durable CAS belong to the caller. + +Inputs are trusted controller records, not authenticated executor messages. This module +neither starts work nor grants a claim, releases a lease, spends budget or promotes a +production version. See docs/speculative-search.md for the adapter boundary. +""" + +from dataclasses import dataclass, fields, replace +from math import isfinite + +from inferrsi.contracts.claim import Verdict + + +def _text(value): + if not isinstance(value, str) or not value.strip(): + raise ValueError("Identity must be a nonempty string") + + +def _integer(value, minimum=0): + if type(value) is not int or value < minimum: + raise ValueError("Expected a bounded nonnegative integer") + + +def _sha(value): + if not isinstance(value, str) or len(value) != 64 or set(value) - set("0123456789abcdef"): + raise ValueError("Expected a lowercase SHA-256 digest") + + +@dataclass(frozen=True) +class Context: + contract: str + plan: str + validator: str + environment: str + workload: str + baseline: str + quality_reference: str + + def __post_init__(self): + for field in fields(self): + _text(getattr(self, field.name)) + + +@dataclass(frozen=True, order=True) +class Dependency: + key: str + epoch: int + + def __post_init__(self): + _text(self.key) + _integer(self.epoch) + + +def _dependencies(values): + values = tuple(values) + if any(not isinstance(value, Dependency) for value in values): + raise ValueError("Expected dependency identities") + if len({value.key for value in values}) != len(values): + raise ValueError("Duplicate dependency key") + return tuple(sorted(values)) + + +@dataclass(frozen=True) +class Producer: + snapshot: str + revision: str + context_digest: str + history_digest: str + + def __post_init__(self): + _text(self.snapshot) + _text(self.revision) + _sha(self.context_digest) + _sha(self.history_digest) + + +@dataclass(frozen=True) +class Contract: + search_id: str + context: Context + improver_snapshot: str + checks: tuple[str, ...] + objective_check: str + direction: str + max_candidates: int + max_depth: int + max_attempts: int + + def __post_init__(self): + _text(self.search_id) + _text(self.improver_snapshot) + if not isinstance(self.context, Context): + raise ValueError("Expected verification context") + if not isinstance(self.checks, (tuple, list)): + raise ValueError("Required checks must be an ordered sequence") + checks = tuple(self.checks) + for check in checks: + _text(check) + if not checks or len(set(checks)) != len(checks) or self.objective_check not in checks: + raise ValueError("Required checks must be unique and include the objective") + object.__setattr__(self, "checks", checks) + if self.direction not in ("maximize", "minimize"): + raise ValueError("Unknown objective direction") + for value in (self.max_candidates, self.max_depth, self.max_attempts): + _integer(value, 1) + + +@dataclass(frozen=True) +class Candidate: + candidate_id: str + artifact_digest: str + parent_id: str | None + producer: Producer + dependencies: tuple[Dependency, ...] = () + + def __post_init__(self): + _text(self.candidate_id) + _sha(self.artifact_digest) + if self.parent_id is not None: + _text(self.parent_id) + if not isinstance(self.producer, Producer): + raise ValueError("Expected producer identity") + object.__setattr__(self, "dependencies", _dependencies(self.dependencies)) + + +@dataclass(frozen=True) +class Attempt: + attempt_id: str + execution_key: str + reservation_id: str + lease_id: str + search_id: str + candidate_id: str + artifact_digest: str + check_id: str + context: Context + dependencies: tuple[Dependency, ...] = () + + def __post_init__(self): + for name in ("attempt_id", "execution_key", "reservation_id", "lease_id", "search_id", + "candidate_id", "check_id"): + _text(getattr(self, name)) + _sha(self.artifact_digest) + if not isinstance(self.context, Context): + raise ValueError("Expected verification context") + object.__setattr__(self, "dependencies", _dependencies(self.dependencies)) + + +@dataclass(frozen=True) +class Receipt: + attempt: Attempt + verdict: Verdict + evidence_digest: str + objective: float | None = None + + def __post_init__(self): + if not isinstance(self.attempt, Attempt) or not isinstance(self.verdict, Verdict): + raise ValueError("Expected bound attempt and verdict") + _sha(self.evidence_digest) + if self.objective is not None and ( + type(self.objective) not in (int, float) or not isfinite(self.objective) + ): + raise ValueError("Objective must be a finite number, not missing-value imputation") + if self.verdict != Verdict.PASS and self.objective is not None: + raise ValueError("Rejected or inconclusive evidence cannot supply an objective") + + +@dataclass(frozen=True) +class SearchState: + contract: Contract + root_id: str + candidates: tuple[Candidate, ...] + attempts: tuple[Attempt, ...] + receipts: tuple[Receipt, ...] + dependencies: tuple[Dependency, ...] + revision: int = 0 + + +def candidate(state, candidate_id): + for value in state.candidates: + if value.candidate_id == candidate_id: + return value + raise ValueError("Unknown candidate") + + +def _current(state, dependencies): + epochs = {item.key: item.epoch for item in state.dependencies} + return all(epochs.get(item.key) == item.epoch for item in dependencies) + + +def candidate_valid(state, candidate_id): + return _current(state, candidate(state, candidate_id).dependencies) + + +def _binding(state, attempt): + node = candidate(state, attempt.candidate_id) + if (attempt.search_id != state.contract.search_id + or attempt.artifact_digest != node.artifact_digest + or attempt.context != state.contract.context + or attempt.check_id not in state.contract.checks + or not set(node.dependencies).issubset(attempt.dependencies)): + raise ValueError("Attempt does not bind this candidate and verification contract") + + +def latest_attempt(state, candidate_id, check_id): + return next((item for item in reversed(state.attempts) + if (item.candidate_id, item.check_id) == (candidate_id, check_id)), None) + + +def evidence(state, candidate_id): + """Current, scoped outcomes, including FAIL; INCONCLUSIVE remains missing.""" + if not candidate_valid(state, candidate_id): + return {} + receipts = {item.attempt.attempt_id: item for item in state.receipts} + result = {} + for check in state.contract.checks: + attempt = latest_attempt(state, candidate_id, check) + if attempt is None or not _current(state, attempt.dependencies): + continue + receipt = receipts.get(attempt.attempt_id) + if receipt is not None and receipt.verdict != Verdict.INCONCLUSIVE: + result[check] = receipt + return result + + +def missing_checks(state, candidate_id): + observed = evidence(state, candidate_id) + return tuple(check for check in state.contract.checks if check not in observed) + + +def complete_candidates(state): + result = [] + for node in state.candidates: + records = evidence(state, node.candidate_id) + if len(records) == len(state.contract.checks) and all( + record.verdict == Verdict.PASS for record in records.values() + ): + result.append(node) + return tuple(result) + + +def accept(state): + """Select a development candidate only; the root wins ties, then stable ID order.""" + complete = complete_candidates(state) + if state.root_id not in {item.candidate_id for item in complete}: + raise ValueError("Search root no longer has complete admissible evidence") + sign = -1 if state.contract.direction == "maximize" else 1 + return min(complete, key=lambda item: ( + sign * evidence(state, item.candidate_id)[state.contract.objective_check].objective, + item.candidate_id != state.root_id, item.candidate_id, + )) + + +def verified_gain(state): + selected = accept(state) + check = state.contract.objective_check + gain = (evidence(state, selected.candidate_id)[check].objective + - evidence(state, state.root_id)[check].objective) + return gain if state.contract.direction == "maximize" else -gain + + +def _receipt_shape(state, receipt): + objective = receipt.attempt.check_id == state.contract.objective_check + if (receipt.verdict == Verdict.PASS and objective) != (receipt.objective is not None): + raise ValueError("Only a passing objective check must supply an objective") + + +def initialize(contract, root, receipts, *, dependencies=()): + """Import previously acquired root evidence; its execution/cost remains caller-owned.""" + receipts = tuple(receipts) + if root.parent_id is not None: + raise ValueError("Search root cannot have an edit parent in this search") + state = SearchState(contract, root.candidate_id, (root,), + tuple(item.attempt for item in receipts), receipts, + _dependencies(dependencies)) + if len(receipts) != len(contract.checks) or len(receipts) > contract.max_attempts: + raise ValueError("Root requires exactly one receipt per required check within the cap") + if len({item.attempt.check_id for item in receipts}) != len(receipts): + raise ValueError("Duplicate root check") + for field in ("attempt_id", "execution_key"): + if len({getattr(item.attempt, field) for item in receipts}) != len(receipts): + raise ValueError("Root execution identities must be unique") + for receipt in receipts: + _binding(state, receipt.attempt) + _receipt_shape(state, receipt) + accept(state) + return state + + +def _guard(state, expected_revision, *, require_root=True): + _integer(expected_revision) + if state.revision != expected_revision: + raise ValueError("Stale search revision; reread and recompute before storage CAS") + if require_root: + accept(state) + + +def add_candidate(state, node, *, expected_revision): + _guard(state, expected_revision) + if len(state.candidates) >= state.contract.max_candidates: + raise ValueError("Candidate limit exhausted") + if any(item.candidate_id == node.candidate_id for item in state.candidates): + raise ValueError("Candidate identity is immutable") + parent = candidate(state, node.parent_id) + if (node.producer.snapshot != state.contract.improver_snapshot + or not _current(state, node.dependencies) + or not set(parent.dependencies).issubset(node.dependencies)): + raise ValueError("Invalid generation dependencies or changed improver snapshot") + depth = 1 + while parent.parent_id is not None: + parent = candidate(state, parent.parent_id) + depth += 1 + if depth > state.contract.max_depth: + raise ValueError("Draft depth limit exhausted") + return replace(state, candidates=(*state.candidates, node), revision=state.revision + 1) + + +def register_attempt(state, attempt, *, expected_revision): + """Record a caller-owned claim. A return value is NOT permission to execute.""" + _guard(state, expected_revision) + _binding(state, attempt) + if not _current(state, attempt.dependencies): + raise ValueError("Attempt depends on invalidated evidence or generation sources") + if len(state.attempts) >= state.contract.max_attempts: + raise ValueError("Check attempt limit exhausted") + if any(item.attempt_id == attempt.attempt_id or item.execution_key == attempt.execution_key + for item in state.attempts): + raise ValueError("Attempt and execution identities cannot be reused") + previous = latest_attempt(state, attempt.candidate_id, attempt.check_id) + if previous is not None: + receipt = next((item for item in state.receipts if item.attempt == previous), None) + if receipt is None: + raise ValueError("Unsettled attempt cannot be replaced or implicitly replayed") + if _current(state, previous.dependencies) and receipt.verdict != Verdict.INCONCLUSIVE: + raise ValueError("A valid outcome cannot be retried until it passes") + return replace(state, attempts=(*state.attempts, attempt), revision=state.revision + 1) + + +def record_receipt(state, receipt, *, expected_revision): + """Retain late receipts as history; evidence() separately decides current admission.""" + _guard(state, expected_revision, require_root=False) + if receipt.attempt not in state.attempts: + raise ValueError("Receipt does not match a registered attempt") + _receipt_shape(state, receipt) + previous = next((item for item in state.receipts + if item.attempt.attempt_id == receipt.attempt.attempt_id), None) + if previous is not None: + if previous != receipt: + raise ValueError("Conflicting receipt for an immutable attempt") + return state + return replace(state, receipts=(*state.receipts, receipt), revision=state.revision + 1) + + +def invalidate(state, dependency_key, *, expected_revision): + """Invalidate a real source, not a parent quality outcome; no cancel/release is issued.""" + _guard(state, expected_revision, require_root=False) + if dependency_key not in {item.key for item in state.dependencies}: + raise ValueError("Unknown dependency") + dependencies = tuple(replace(item, epoch=item.epoch + 1) if item.key == dependency_key + else item for item in state.dependencies) + return replace(state, dependencies=dependencies, revision=state.revision + 1) diff --git a/src/inferrsi/harness/verification_plans.py b/src/inferrsi/harness/verification_plans.py new file mode 100644 index 00000000..81d39f86 --- /dev/null +++ b/src/inferrsi/harness/verification_plans.py @@ -0,0 +1,145 @@ +"""Bounded exact completion-menu construction for offline VATS controls. + +Costs are caller-supplied conservative integer bounds in one declared unit. A plan +is a proposal against a budget view, never a resource reservation or execution grant. +""" + +from dataclasses import dataclass +from itertools import combinations + +from inferrsi.contracts.claim import Verdict + +from .search_tree import accept, candidate_valid, evidence, missing_checks + + +@dataclass(frozen=True) +class Job: + job_id: str + cost: int + candidate_id: str | None = None + check_id: str | None = None + prerequisites: tuple[str, ...] = () + conflicts: tuple[str, ...] = () + + def __post_init__(self): + if not isinstance(self.job_id, str) or not self.job_id.strip(): + raise ValueError("Job identity must be nonempty") + if type(self.cost) is not int or self.cost <= 0: + raise ValueError("Job cost must be a known positive integer upper bound") + if (self.candidate_id is None) != (self.check_id is None): + raise ValueError("A check job needs both candidate and check identities") + for value in (self.candidate_id, self.check_id): + if value is not None and (not isinstance(value, str) or not value.strip()): + raise ValueError("Check identity must be nonempty") + for name in ("prerequisites", "conflicts"): + if not isinstance(getattr(self, name), (tuple, list)): + raise ValueError("Job relations must be ordered sequences") + values = tuple(getattr(self, name)) + if (any(not isinstance(value, str) or not value.strip() for value in values) + or len(set(values)) != len(values) or self.job_id in values): + raise ValueError("Job relations must be unique, nonempty, non-self identities") + object.__setattr__(self, name, values) + + +@dataclass(frozen=True) +class Plan: + candidate_ids: tuple[str, ...] + job_ids: tuple[str, ...] + cost: int + search_id: str + state_revision: int + + +def completion_menu(state, jobs, *, available_budget, available_jobs, max_targets, max_jobs): + """Enumerate unions of up to max_targets completion seeds, retaining every feasible seed. + +The caller must settle a synchronous work block first. Preparation jobs have no +candidate/check pair. Shared prerequisites are charged once per proposed plan; +conflicts prohibit co-scheduling even when only one side declares the relation. +""" + for value, minimum in ((available_budget, 0), (available_jobs, 0), + (max_targets, 1), (max_jobs, 1)): + if type(value) is not int or value < minimum: + raise ValueError("Menu limits and available budget must be known integers") + accept(state) + receipts = {item.attempt.attempt_id for item in state.receipts} + # An invalidated in-flight job is still real work; only its owner can settle it. + if any(item.attempt_id not in receipts for item in state.attempts): + raise ValueError("Settle the current work block before constructing another menu") + jobs = tuple(jobs) + if len(jobs) > max_jobs: + raise ValueError("Job menu exceeds the declared enumeration bound") + by_id = {job.job_id: job for job in jobs} + if len(by_id) != len(jobs): + raise ValueError("Duplicate job identity") + checks = {} + for job in jobs: + if (set(job.prerequisites) | set(job.conflicts)) - by_id.keys(): + raise ValueError("Unknown job relation") + if job.candidate_id is not None: + candidate_valid(state, job.candidate_id) # Reject unknown candidate IDs. + if job.check_id not in state.contract.checks: + raise ValueError("Unknown required check") + pair = (job.candidate_id, job.check_id) + if pair in checks: + raise ValueError("Ambiguous jobs for one candidate check") + checks[pair] = job.job_id + + closures = {} + + def closure(job_id, visiting=frozenset()): + if job_id in visiting: + raise ValueError("Cyclic job prerequisites") + if job_id not in closures: + result = {job_id} + for parent in by_id[job_id].prerequisites: + result.update(closure(parent, visiting | {job_id})) + closures[job_id] = frozenset(result) + return closures[job_id] + + for job in jobs: + closure(job.job_id) + + seeds = {} + for node in state.candidates: + cid = node.candidate_id + if not candidate_valid(state, cid) or any( + item.verdict == Verdict.FAIL for item in evidence(state, cid).values() + ): + continue + missing = missing_checks(state, cid) + if not missing or any((cid, check) not in checks for check in missing): + continue + seeds[cid] = frozenset().union(*(closure(checks[(cid, check)]) for check in missing)) + + plans = {} + remaining_attempts = state.contract.max_attempts - len(state.attempts) + for size in range(1, min(max_targets, len(seeds)) + 1): + for targets in combinations(sorted(seeds), size): + union = frozenset().union(*(seeds[cid] for cid in targets)) + if len(union) > available_jobs: + continue + check_jobs = [by_id[jid] for jid in union if by_id[jid].candidate_id is not None] + if len(check_jobs) > remaining_attempts: + continue + # A cross-candidate prerequisite must itself be a legal outstanding check. + if any(job.check_id not in missing_checks(state, job.candidate_id) + or not candidate_valid(state, job.candidate_id) for job in check_jobs): + continue + if any(set(by_id[jid].conflicts) & union for jid in union): + continue + cost = sum(by_id[jid].cost for jid in union) + if cost > available_budget: + continue + # Stable topological order, so an adapter never receives a child before setup. + ordered = [] + remaining = set(union) + while remaining: + ready = sorted(jid for jid in remaining + if not (set(by_id[jid].prerequisites) & remaining)) + ordered.extend(ready) + remaining.difference_update(ready) + completed = tuple(sorted(cid for cid, seed in seeds.items() if seed <= union)) + plan = Plan(completed, tuple(ordered), cost, state.contract.search_id, state.revision) + plans[plan.job_ids] = plan + return tuple(sorted(plans.values(), key=lambda plan: (plan.cost, plan.job_ids))) diff --git a/tests/fixtures/speculative_search.py b/tests/fixtures/speculative_search.py new file mode 100644 index 00000000..10493820 --- /dev/null +++ b/tests/fixtures/speculative_search.py @@ -0,0 +1,67 @@ +"""Synthetic observations and identities; no model, hardware or billing evidence.""" + +from dataclasses import replace +from hashlib import sha256 + +from inferrsi.contracts.claim import Verdict +from inferrsi.harness import search_tree as tree + + +def digest(text): + return sha256(text.encode()).hexdigest() + + +def contract(**changes): + return replace(tree.Contract( + "search", tree.Context("contract", "plan", "grader", "cpu-fixture", "development", + "original-baseline", "quality-reference"), + "improver-0", ("quality", "objective"), "objective", "maximize", 20, 5, 40, + ), **changes) + + +def producer(snapshot="improver-0"): + return tree.Producer(snapshot, "synthetic-producer-1", digest("context"), digest("history")) + + +def attempt(state, cid, check, *, dependencies=(), tag=None): + node = tree.candidate(state, cid) + tag = tag or str(len(state.attempts)) + return tree.Attempt( + f"attempt-{tag}", f"execution-{tag}", f"reservation-{tag}", f"lease-{tag}", + state.contract.search_id, cid, node.artifact_digest, check, state.contract.context, + tuple(sorted(set(node.dependencies) | set(dependencies))), + ) + + +def receipt(binding, verdict=Verdict.PASS, objective=None): + return tree.Receipt(binding, verdict, digest(f"receipt:{binding.attempt_id}"), objective) + + +def start(*, configuration=None, dependencies=(), root_dependencies=(), score=1): + configuration = configuration or contract() + root = tree.Candidate("root", digest("root"), None, producer(), root_dependencies) + seed = tree.SearchState(configuration, "root", (root,), (), (), tuple(dependencies)) + receipts = tuple(receipt(attempt(seed, "root", check, tag=f"root-{check}"), + objective=score if check == configuration.objective_check else None) + for check in configuration.checks) + return tree.initialize(configuration, root, receipts, dependencies=dependencies) + + +def add(state, cid, parent="root", *, dependencies=None, artifact_digest=None, snapshot=None): + node = tree.Candidate(cid, artifact_digest or digest(cid), parent, + producer(snapshot or state.contract.improver_snapshot), + tree.candidate(state, parent).dependencies if dependencies is None + else dependencies) + return tree.add_candidate(state, node, expected_revision=state.revision) + + +def observe(state, cid, check, verdict=Verdict.PASS, objective=None, *, dependencies=()): + binding = attempt(state, cid, check, dependencies=dependencies) + state = tree.register_attempt(state, binding, expected_revision=state.revision) + return tree.record_receipt(state, receipt(binding, verdict, objective), + expected_revision=state.revision) + + +def complete(state, cid, score): + state = observe(state, cid, "quality") + return observe(state, cid, "objective", objective=score) diff --git a/tests/test_speculative_search.py b/tests/test_speculative_search.py new file mode 100644 index 00000000..997e95f5 --- /dev/null +++ b/tests/test_speculative_search.py @@ -0,0 +1,263 @@ +from dataclasses import FrozenInstanceError, fields, replace + +import pytest + +from inferrsi.contracts.claim import Verdict +from inferrsi.harness import candidates +from inferrsi.harness import search_tree as tree +from tests.fixtures.speculative_search import ( + add, + attempt, + complete, + contract, + digest, + observe, + producer, + receipt, + start, +) + + +@pytest.mark.parametrize("lane,filename", [("engine", "kernel.py"), ("skill", "SKILL.md")]) +def test_failed_parent_can_be_repaired_with_its_own_snapshot_and_complete_evidence( + tmp_path, lane, filename, +): + # File retention is real; grades are synthetic controls, not either production adapter. + work = tmp_path / "work" + work.mkdir() + source = work / filename + state = start(configuration=contract(search_id=lane)) + for cid, value, parent in (("broken", -1, "root"), ("repaired", 3, "broken")): + source.write_text(str(value)) + retained = tmp_path / cid + identity = candidates.retain(work, retained) + state = add(state, cid, parent, artifact_digest=identity) + actual = int((retained / filename).read_text()) + state = observe(state, cid, "quality", Verdict.PASS if actual > 0 else Verdict.FAIL) + if actual > 0: + # A high objective alone never compensates for missing or failed quality. + state = observe(state, cid, "objective", objective=actual) + source.write_text("999") + assert candidates.identity(tmp_path / "repaired") == tree.accept(state).artifact_digest + assert tree.accept(state).candidate_id == "repaired" + assert tree.verified_gain(state) == 2 + assert tree.missing_checks(state, "broken") == ("objective",) + assert tree.evidence(state, "broken")["quality"].verdict == Verdict.FAIL + assert tree.candidate_valid(state, "broken") + + +def test_incomplete_parent_can_be_extended_without_fabricating_parent_results(): + state = add(add(start(), "parent"), "child", "parent") + state = complete(state, "child", 4) + assert tree.accept(state).candidate_id == "child" + assert tree.evidence(state, "parent") == {} + + +def test_objective_prediction_partial_results_and_failed_quality_cannot_win(): + state = add(start(), "candidate") + state = observe(state, "candidate", "objective", objective=1000) + assert tree.accept(state).candidate_id == "root" + state = observe(state, "candidate", "quality", Verdict.FAIL) + assert tree.missing_checks(state, "candidate") == () # Known failure is not missing. + assert tree.accept(state).candidate_id == "root" + assert tree.verified_gain(state) == 0 + + +@pytest.mark.parametrize("direction", ["maximize", "minimize"]) +@pytest.mark.parametrize("order", [("z", "a"), ("a", "z")]) +def test_selection_is_independent_of_receipt_order_and_root_wins_ties(direction, order): + state = start(configuration=contract(direction=direction), score=10) + for cid in order: + state = complete(add(state, cid), cid, 11 if direction == "maximize" else 9) + assert tree.accept(state).candidate_id == "a" + assert tree.verified_gain(state) == 1 + tied = complete(add(start(score=10), "a"), "a", 10) + assert tree.accept(tied).candidate_id == "root" + + +@pytest.mark.parametrize("late", [True, False]) +def test_real_dependency_invalidation_and_success_race_cannot_leave_stale_incumbent(late): + dependency = tree.Dependency("measurement-source", 0) + state = add(start(dependencies=(dependency,)), "candidate") + state = observe(state, "candidate", "quality") + binding = attempt(state, "candidate", "objective", dependencies=(dependency,)) + state = tree.register_attempt(state, binding, expected_revision=state.revision) + result = receipt(binding, objective=5) + if not late: + state = tree.record_receipt(state, result, expected_revision=state.revision) + assert tree.accept(state).candidate_id == "candidate" + state = tree.invalidate(state, dependency.key, expected_revision=state.revision) + if late: + state = tree.record_receipt(state, result, expected_revision=state.revision) + assert result in state.receipts # Archive is distinct from evidence admission. + assert tree.accept(state).candidate_id == "root" + assert tree.missing_checks(state, "candidate") == ("objective",) + + +def test_old_receipt_after_new_attempt_cannot_override_current_outcome_or_be_reused(): + dependency = tree.Dependency("measurement-source", 0) + state = add(start(dependencies=(dependency,)), "candidate") + state = observe(state, "candidate", "quality") + state = observe(state, "candidate", "objective", objective=9, dependencies=(dependency,)) + old = state.receipts[-1] + state = tree.invalidate(state, dependency.key, expected_revision=state.revision) + new_dependency = state.dependencies[0] + binding = attempt(state, "candidate", "objective", dependencies=(new_dependency,)) + state = tree.register_attempt(state, binding, expected_revision=state.revision) + state = tree.record_receipt(state, old, expected_revision=state.revision) + assert tree.accept(state).candidate_id == "root" + state = tree.record_receipt(state, receipt(binding, Verdict.FAIL), + expected_revision=state.revision) + before = state + state = tree.record_receipt(state, old, expected_revision=state.revision) + assert state is before + assert tree.evidence(state, "candidate")["objective"].verdict == Verdict.FAIL + assert tree.accept(state).candidate_id == "root" + + +def test_generation_invalidation_affects_actual_descendants_but_not_other_branch(): + dependency = tree.Dependency("retrieved-context", 0) + state = start(dependencies=(dependency,)) + state = add(state, "parent", dependencies=(dependency,)) + state = complete(add(state, "child", "parent"), "child", 10) + state = complete(add(state, "other"), "other", 3) + state = tree.invalidate(state, dependency.key, expected_revision=state.revision) + assert not tree.candidate_valid(state, "child") + assert tree.accept(state).candidate_id == "other" + with pytest.raises(ValueError, match="dependencies"): + add(state, "grandchild", "child", dependencies=()) + + +def test_invalid_root_blocks_search_and_acceptance_but_allows_late_archival(): + dependency = tree.Dependency("root-source", 0) + state = add(start(dependencies=(dependency,), root_dependencies=(dependency,)), "candidate") + binding = attempt(state, "candidate", "quality") + state = tree.register_attempt(state, binding, expected_revision=state.revision) + state = tree.invalidate(state, dependency.key, expected_revision=state.revision) + state = tree.record_receipt(state, receipt(binding), expected_revision=state.revision) + with pytest.raises(ValueError, match="root"): + tree.accept(state) + with pytest.raises(ValueError, match="root"): + add(state, "another") + + +def test_inconclusive_is_missing_and_may_be_retried_with_fresh_identity(): + state = add(start(), "candidate") + state = observe(state, "candidate", "quality", Verdict.INCONCLUSIVE) + assert tree.missing_checks(state, "candidate") == ("quality", "objective") + state = complete(state, "candidate", 2) + assert tree.accept(state).candidate_id == "candidate" + assert len([item for item in state.attempts if item.check_id == "quality"]) == 3 + + +@pytest.mark.parametrize("verdict", [Verdict.PASS, Verdict.FAIL]) +def test_valid_outcomes_cannot_be_retried_until_they_pass(verdict): + state = observe(add(start(), "candidate"), "candidate", "quality", verdict) + with pytest.raises(ValueError, match="retried"): + observe(state, "candidate", "quality") + + +def test_unsettled_attempt_blocks_replacement_even_after_invalidation(): + dependency = tree.Dependency("measurement-source", 0) + state = add(start(dependencies=(dependency,)), "candidate") + binding = attempt(state, "candidate", "quality", dependencies=(dependency,)) + state = tree.register_attempt(state, binding, expected_revision=state.revision) + state = tree.invalidate(state, dependency.key, expected_revision=state.revision) + replacement = attempt(state, "candidate", "quality", dependencies=state.dependencies) + with pytest.raises(ValueError, match="Unsettled"): + tree.register_attempt(state, replacement, expected_revision=state.revision) + + +@pytest.mark.parametrize("field", [field.name for field in fields(tree.Context)]) +def test_receipts_from_changed_contract_plan_grader_environment_or_data_are_rejected(field): + state = add(start(), "candidate") + binding = attempt(state, "candidate", "quality") + state = tree.register_attempt(state, binding, expected_revision=state.revision) + wrong = replace(binding, context=replace(binding.context, **{field: "different"})) + with pytest.raises(ValueError, match="registered"): + tree.record_receipt(state, receipt(wrong), expected_revision=state.revision) + + +@pytest.mark.parametrize("field,value", [ + ("search_id", "other-search"), ("candidate_id", "root"), + ("artifact_digest", digest("different")), ("check_id", "objective"), + ("attempt_id", "other-attempt"), ("execution_key", "other-execution"), + ("reservation_id", "other-reservation"), ("lease_id", "other-lease"), +]) +def test_receipt_must_match_the_entire_registered_binding(field, value): + state = add(start(), "candidate") + binding = attempt(state, "candidate", "quality") + state = tree.register_attempt(state, binding, expected_revision=state.revision) + with pytest.raises(ValueError, match="registered"): + tree.record_receipt(state, receipt(replace(binding, **{field: value})), + expected_revision=state.revision) + + +def test_same_bytes_do_not_transfer_evidence_and_conflicting_receipts_are_rejected(): + state = complete(add(start(), "first", artifact_digest=digest("same")), "first", 5) + old = state.receipts[-1] + state = add(state, "second", "first", artifact_digest=digest("same")) + assert tree.evidence(state, "second") == {} + with pytest.raises(ValueError, match="Conflicting"): + tree.record_receipt(state, replace(old, objective=6), expected_revision=state.revision) + with pytest.raises(ValueError, match="reused"): + tree.register_attempt(state, replace(old.attempt, candidate_id="second"), + expected_revision=state.revision) + + +def test_generation_snapshot_candidates_and_history_are_immutable(): + state = start() + with pytest.raises(ValueError, match="snapshot"): + add(state, "candidate", snapshot="updated-mid-search") + advanced = add(state, "candidate") + assert len(state.candidates) == 1 + with pytest.raises(FrozenInstanceError): + advanced.candidates[1].artifact_digest = digest("mutated") + with pytest.raises(ValueError, match="immutable"): + add(advanced, "candidate", artifact_digest=digest("changed")) + with pytest.raises(ValueError, match="Stale"): + tree.add_candidate(advanced, tree.Candidate("other", digest("other"), "root", producer()), + expected_revision=state.revision) + + +@pytest.mark.parametrize("limit", ["max_candidates", "max_depth", "max_attempts"]) +def test_bounded_search_still_returns_the_qualified_root(limit): + config = contract(**{limit: 2 if limit == "max_attempts" else 1}) + state = start(configuration=config) + with pytest.raises(ValueError, match="limit"): + if limit == "max_candidates": + add(state, "first") + elif limit == "max_depth": + add(add(state, "first"), "second", "first") + else: + observe(add(state, "first"), "first", "quality") + assert tree.accept(state).candidate_id == "root" + + +@pytest.mark.parametrize("value", [float("nan"), float("inf"), True, "10"]) +def test_invalid_objective_never_becomes_evidence(value): + state = add(start(), "candidate") + with pytest.raises(ValueError, match="finite"): + receipt(attempt(state, "candidate", "objective"), objective=value) + + +def test_root_bootstrap_requires_complete_scoped_passing_evidence(): + state = start() + for receipts in (state.receipts[:1], (*state.receipts, state.receipts[0]), + (replace(state.receipts[0], verdict=Verdict.FAIL), state.receipts[1])): + with pytest.raises(ValueError): + tree.initialize(state.contract, state.candidates[0], receipts) + with pytest.raises(ValueError, match="objective"): + tree.initialize(state.contract, state.candidates[0], + (state.receipts[0], replace(state.receipts[1], objective=None))) + + +def test_contract_copies_ordered_checks_and_rejects_strings_and_unknown_dependencies(): + checks = ["quality", "objective"] + configuration = contract(checks=checks) + checks.clear() + assert configuration.checks == ("quality", "objective") + with pytest.raises(ValueError, match="ordered"): + contract(checks="quality") + with pytest.raises(ValueError, match="dependencies"): + add(start(), "candidate", dependencies=(tree.Dependency("undeclared", 0),)) diff --git a/tests/test_verification_plans.py b/tests/test_verification_plans.py new file mode 100644 index 00000000..d87028f4 --- /dev/null +++ b/tests/test_verification_plans.py @@ -0,0 +1,151 @@ +from dataclasses import replace +from itertools import combinations + +import pytest + +from inferrsi.contracts.claim import Verdict +from inferrsi.harness import search_tree as tree +from inferrsi.harness.verification_plans import Job, completion_menu +from tests.fixtures.speculative_search import add, attempt, contract, observe, start + + +def menu(state, jobs, **changes): + options = dict(available_budget=100, available_jobs=20, max_targets=3, max_jobs=20) + return completion_menu(state, jobs, **(options | changes)) + + +def fixture(): + state = start() + jobs = [Job("setup", 3)] + for cid in ("a", "b", "c"): + state = add(state, cid) + jobs.extend((Job(f"{cid}-quality", 2, cid, "quality", ("setup",)), + Job(f"{cid}-objective", 4, cid, "objective", (f"{cid}-quality",)))) + return state, tuple(jobs) + + +def test_low_standalone_value_checks_are_preserved_as_completion_bundles(): + state, jobs = fixture() + plans = menu(state, jobs, available_budget=9) + assert len(plans) == 3 + assert {plan.candidate_ids for plan in plans} == {("a",), ("b",), ("c",)} + assert all(len(plan.job_ids) == 3 and plan.cost == 9 for plan in plans) + assert all(plan.job_ids[0] == "setup" for plan in plans) + # No partial check subset can be presented as a completed candidate. + assert menu(state, jobs, available_budget=8) == () + + +def test_cross_branch_completion_counts_shared_preparation_once(): + state, jobs = fixture() + plans = menu(state, jobs, available_budget=15, max_targets=2) + pairs = [plan for plan in plans if len(plan.candidate_ids) == 2] + assert len(pairs) == 3 + assert all(plan.cost == 15 and plan.job_ids.count("setup") == 1 for plan in pairs) + assert all(len(plan.job_ids) == 5 for plan in pairs) + + +def exhaustive_oracle(jobs, budget, job_limit, target_limit): + # Independent oracle enumerates all job subsets, not candidate-completion unions. + result = {} + for size in range(len(jobs) + 1): + for subset in combinations(jobs, size): + ids = {job.job_id for job in subset} + if (len(ids) > job_limit or sum(job.cost for job in subset) > budget + or any(set(job.prerequisites) - ids or set(job.conflicts) & ids + for job in subset)): + continue + targets = tuple(cid for cid in ("a", "b", "c") + if {f"{cid}-quality", f"{cid}-objective"} <= ids) + if not 0 < len(targets) <= target_limit: + continue + cost = sum(job.cost for job in subset) + result[targets] = min(cost, result.get(targets, cost)) + return result + + +@pytest.mark.parametrize("budget", [0, 8, 9, 14, 15, 20, 21]) +@pytest.mark.parametrize("target_limit", [1, 2, 3]) +def test_exact_menu_matches_independent_exhaustive_job_subset_oracle(budget, target_limit): + state, jobs = fixture() + expected = exhaustive_oracle(jobs, budget, 7, target_limit) + actual = {plan.candidate_ids: plan.cost for plan in menu( + state, jobs, available_budget=budget, available_jobs=7, max_targets=target_limit, + )} + assert actual == expected + + +def test_conflicts_job_caps_and_attempt_caps_cannot_be_overridden_by_budget(): + state, jobs = fixture() + jobs = tuple(replace(job, conflicts=("b-objective",)) if job.job_id == "a-objective" + else job for job in jobs) + assert all(not {"a", "b"} <= set(plan.candidate_ids) for plan in menu(state, jobs)) + assert menu(state, jobs, available_jobs=2) == () + limited = replace(state, contract=replace(state.contract, max_attempts=3)) + assert menu(limited, jobs) == () # Root used two; completing another needs two. + with pytest.raises(ValueError, match="enumeration"): + menu(state, jobs, max_jobs=6) + + +def test_partial_evidence_reduces_work_but_failure_is_not_a_completion_target(): + state, jobs = fixture() + state = observe(state, "a", "quality") + state = observe(state, "b", "quality", Verdict.FAIL) + # The adapter rebuilds the menu from remaining work and scopes prerequisite reuse. + jobs = tuple(replace(job, prerequisites=("setup",)) if job.job_id == "a-objective" + else job for job in jobs if job.job_id != "a-quality") + plans = menu(state, jobs, available_budget=7) + assert [(plan.candidate_ids, plan.cost) for plan in plans] == [(("a",), 7)] + + +def test_unknown_cost_and_invalid_dependency_graphs_are_rejected(): + state, jobs = fixture() + for cost in (0, -1, True, None, float("nan")): + with pytest.raises(ValueError, match="cost"): + Job("unknown", cost) + with pytest.raises(ValueError, match="Unknown job relation"): + menu(state, (*jobs, Job("orphan", 1, prerequisites=("absent",)))) + with pytest.raises(ValueError, match="Cyclic"): + menu(state, (*jobs, Job("x", 1, prerequisites=("y",)), + Job("y", 1, prerequisites=("x",)))) + with pytest.raises(ValueError, match="Ambiguous"): + menu(state, (*jobs, Job("duplicate", 1, "a", "quality"))) + with pytest.raises(ValueError, match="ordered"): + Job("x", 1, prerequisites="setup") + + +def test_inflight_work_prevents_replanning_even_if_source_is_invalidated(): + dependency = tree.Dependency("source", 0) + state = add(start(dependencies=(dependency,)), "a") + binding = attempt(state, "a", "quality", dependencies=(dependency,)) + state = tree.register_attempt(state, binding, expected_revision=state.revision) + state = tree.invalidate(state, dependency.key, expected_revision=state.revision) + with pytest.raises(ValueError, match="Settle"): + menu(state, []) + + +def test_inconclusive_outcome_remains_a_completion_job(): + state, jobs = fixture() + state = observe(state, "a", "quality", Verdict.INCONCLUSIVE) + assert ("a",) in {plan.candidate_ids for plan in menu(state, jobs, available_budget=9)} + + +def test_planning_never_changes_evidence_or_spends_the_supplied_budget(): + state, jobs = fixture() + before = state + plans = menu(state, jobs) + assert plans + assert all((plan.search_id, plan.state_revision) == (state.contract.search_id, state.revision) + for plan in plans) + assert state is before + assert tree.accept(state).candidate_id == "root" + assert tree.verified_gain(state) == 0 + with pytest.raises(ValueError, match="known integers"): + menu(state, jobs, available_budget=None) + + +def test_all_jobs_must_name_known_candidates_and_frozen_required_checks(): + state = start(configuration=contract()) + with pytest.raises(ValueError, match="Unknown candidate"): + menu(state, [Job("q", 1, "absent", "quality")]) + with pytest.raises(ValueError, match="Unknown required check"): + menu(state, [Job("q", 1, "root", "unregistered-check")]) From abe72f633747e86c8a2b59e4b6e117fe39bf204c Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Wed, 18 Nov 2026 21:12:59 +0800 Subject: [PATCH 2/8] harness: assemble bounded VATS and GVO with durable work control Signed-off-by: amy-why-3459 --- docs/architecture.md | 11 +- docs/module-layout.md | 3 + docs/operations.md | 3 + docs/speculative-search.md | 356 ++++++---- src/inferrsi/apps/cli/__init__.py | 5 + src/inferrsi/apps/cli/arguments.py | 3 + src/inferrsi/apps/search_benchmarks.py | 159 +++++ src/inferrsi/apps/speculative_search.py | 218 ++++++ src/inferrsi/harness/gvo.py | 213 ++++++ src/inferrsi/harness/local_process.py | 7 +- src/inferrsi/harness/scenarios/external.py | 3 + src/inferrsi/harness/search_adapter.py | 283 ++++++++ src/inferrsi/harness/search_campaign.py | 341 ++++++++++ src/inferrsi/harness/search_continuations.py | 155 +++++ src/inferrsi/harness/search_records.py | 60 ++ src/inferrsi/harness/search_tree.py | 2 + src/inferrsi/harness/vats.py | 659 +++++++++++++++++++ src/inferrsi/harness/work_control.py | 278 ++++++++ src/inferrsi/model/search_roles.py | 215 ++++++ tests/test_search_benchmarks.py | 115 ++++ tests/test_search_roles.py | 71 ++ tests/test_speculative_search.py | 8 + tests/test_vats_runtime.py | 310 +++++++++ tests/test_work_control.py | 152 +++++ tools/speculative_skill.py | 165 +++++ 25 files changed, 3655 insertions(+), 140 deletions(-) create mode 100644 src/inferrsi/apps/search_benchmarks.py create mode 100644 src/inferrsi/apps/speculative_search.py create mode 100644 src/inferrsi/harness/gvo.py create mode 100644 src/inferrsi/harness/search_adapter.py create mode 100644 src/inferrsi/harness/search_campaign.py create mode 100644 src/inferrsi/harness/search_continuations.py create mode 100644 src/inferrsi/harness/search_records.py create mode 100644 src/inferrsi/harness/vats.py create mode 100644 src/inferrsi/harness/work_control.py create mode 100644 src/inferrsi/model/search_roles.py create mode 100644 tests/test_search_benchmarks.py create mode 100644 tests/test_search_roles.py create mode 100644 tests/test_vats_runtime.py create mode 100644 tests/test_work_control.py create mode 100644 tools/speculative_skill.py diff --git a/docs/architecture.md b/docs/architecture.md index e2f05e8e..7bc0a272 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -28,11 +28,12 @@ does not import Benchmark. Benchmark supplies task definitions and specialized rules, while Harness manages their execution and the complete optimization lifecycle. Neither infers quality criteria from a natural-language goal. Ordinary projects default to the current Agent conversation. -The [offline speculative-search primitives](speculative-search.md) add immutable -candidate trees, scoped evidence transitions and bounded completion menus in -Harness. They expose pure control operations for future VATS assembly; the -existing session controller continues to own execution. Production adapter -integration, learned role selection and GVO remain separate delivery work. +The [experimental speculative search](speculative-search.md) composes immutable +candidate trees, bounded completion menus, synchronous VATS and CPU GVO roles. +Its work controller uses shared RunStore transactions and the existing local +process executor. Apps supplies Engine/Skill grading; the support matrix identifies +remaining production accounting and pilot evidence. Serial session commands keep +their existing controller and behavior. The [native host handoff](native-agent-sessions.md) lets the initiating Agent resolve the task request before any campaign exists, use its host's new-session tool and retain the native identity and returned artifacts. The child owns the diff --git a/docs/module-layout.md b/docs/module-layout.md index bfd68633..589d6202 100644 --- a/docs/module-layout.md +++ b/docs/module-layout.md @@ -114,6 +114,9 @@ Several flat modules remain shared seams until their slice starts. | `harness/optimization_session.py`, `harness/project.py`, `harness/task_protocol.py` | Shared preparation, execution and retained-result assessment | CLI supplies task resources and knowledge access; ordinary project lifecycle does not import Benchmark | | `harness/optimization.py`, `harness/candidates.py`, `harness/usage.py` | Shared project and packaged-task lifecycle | Injected execution/assessment; declared file/directory candidates, selection direction and exact/up-to rounds | | `harness/agent_sessions.py` | Native host handoff records | Freezes requests and retains native IDs and exported observations; the initiating Agent calls host tools | +| `harness/search_tree.py`, `verification_plans.py`, `vats.py`, `gvo.py`, `search_campaign.py`, `search_continuations.py` | Experimental candidate search and group learning | Frozen contracts and injected evaluators; CPU roles live in `model/search_roles.py` | +| `harness/work_control.py`, `search_records.py`, `search_adapter.py` | Search persistence, accounting and artifact adapters | Shared RunStore transactions and existing local process executor; trusted local host profile | +| `apps/search_benchmarks.py`, `apps/speculative_search.py` | Search composition and Engine CLI | Join task-owned assessment with Harness control; no benchmark acceptance rules embedded in the search algorithm | | `harness/skill_evaluation.py`, `harness/artifact_sandbox.py`, `harness/improvement.py`, `harness/atomic_json.py` | Skill-evaluation host adapter, trial execution and retention, sandbox replay, search progression | Verdict rules and task definitions arrive as injected values; no Model, Infra, Knowledge, Benchmark or Apps import | | `benchmark/skill_evaluation.py`, `benchmark/skill_eval_diagnostic.py`, `benchmark/skill_eval_engineering.py`, `benchmark/improvement.py` | Skill-evaluation criteria, task definitions and acceptance rules | Pure judgments over retained facts; no Harness import and no process start | | `apps/skill_evaluation.py`, `apps/skill_eval_engineering.py`, `apps/improvement.py` | Composed skill-evaluation entries | Join Knowledge reads, Harness execution and Benchmark rules behind the signatures that `harness/` held before the split | diff --git a/docs/operations.md b/docs/operations.md index 176e0c8c..c750af2f 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -66,6 +66,9 @@ Effect classes used below: | Replay a cloud-generated parser pair | [Isolated cloud replay](skill-eval-cloud-replay.md), `python -m tools.skill_eval_cloud_replay` | Untrusted-artifact execution / report-generating | Manual pinned data ingress; same-host controls before bubblewrap-only replay; no models or credentials | | Validate one harness-improvement step | [Offline improvement](harness-improvement.md), Python `tools.skill_eval_improve.prepare` / `run_offline` | Snapshot-writing / offline experiment-executing | Supplied declarative INV/DOM edits, shared total limits, rerun and one holdout gate; trusted fixed-output clients only; no real-model or Store execution | | Build an offline candidate tree and completion menu | Python `harness.search_tree` / `harness.verification_plans.completion_menu`; [control contract](speculative-search.md) | Read-only / in-memory transitions | Immutable candidates, scoped caller-supplied receipts and bounded exact plans; execution, durable state, accounting, learned roles and final acceptance require separate adapters | +| Run bounded speculative Engine search and role updates | `inferrsi search engine`; [support and configuration](speculative-search.md) | Execution / candidate writes / CPU model updates | Explicit shared RunStore, permissions, checkpoint, budgets, domain and final gate; actual device allocation is caller-owned | +| Inspect/export speculative search | `inferrsi search status` / `search export` | Read / output artifact | Reads the existing store; never restarts search, training or final evaluation | +| Compose Skill speculative search | Python `apps.search_benchmarks.SkillEvaluator` and checkout `tools.speculative_skill.Comparator` | Explicit provider execution through existing comparator | Frozen original baseline, split descriptors, independent grader and bounded provider; see campaign accounting limits in the support matrix | | Plan and report skill PR evaluations | [Skill PR integration](skill-eval-pr.md), `python -m tools.skill_eval_pr` | Read-only planning / authorized experiment-executing / PR-publishing | Trusted mapping, exact operator approval, consumed execution identity and credential-separated publication from retained evidence; no automatic paid trials | | Recover goal progress and the next proposal | [Offline goal checkpoints](goal-checkpoints.md), `python -m tools.goal_checkpoint` | Read-only | Explicit scope/criteria, pinned owner decisions and existing comparison receipts; a disposable view, never execution or acceptance authority | | Summarize campaign gains | `tools/campaign_summary_chart.py`; `report`/`run` figure | Figure-generating | Renders recorded values into one gain figure plus a matching table; validates the specification and computes nothing new | diff --git a/docs/speculative-search.md b/docs/speculative-search.md index 96d6e38a..d7639bb5 100644 --- a/docs/speculative-search.md +++ b/docs/speculative-search.md @@ -1,140 +1,228 @@ -# Offline candidate trees and verification plans - -This is the first implementation slice of -[the SpecRSI proposal](https://github.com/InferMatrixLab/InferRSI/issues/363). -`harness/search_tree.py` provides immutable candidate and evidence transitions; -`harness/verification_plans.py` constructs bounded completion menus. Both are -pure Python control primitives with no execution, storage or model access. -The existing project and benchmark session commands retain their current behavior. - -The design follows [SpecRSI v2, Section 3](https://arxiv.org/html/2610.16491v2#S3): -editing an incomplete or unsuccessful parent is allowed, while adopting a version -requires complete admissible evidence for that version. Completion bundles keep -complementary checks together. This slice supplies building blocks for the inner -search; a learned Scorer/Policy, automatic Drafter and GVO updates remain future -integration work. - -## State and evidence - -`Contract` freezes the search identity, verification context, required checks, -objective, direction, improver snapshot and candidate/depth/check-attempt caps. -The context identifies the plan, validator, environment, workload, original -baseline and quality reference. Checks may encode separately required repetitions. -The current selector supports a scalar objective plus hard PASS requirements; -the Skill lane's quality-first lexicographic selector needs a separate adapter. - -`initialize` requires one complete, passing receipt per root check. Root receipts -count against the attempt cap. Their acquisition/reuse cost must already be in -the caller's authoritative accounting. A passing objective check carries a finite -value; other checks and rejected/inconclusive outcomes cannot supply a value. - -`add_candidate` records an immutable artifact digest and edit parent, with the -frozen producer snapshot, producer revision, context/history digests and actual -generation dependencies. Callers retain and verify artifacts with the existing -`harness.candidates.retain` / `identity` helpers before admission. This core does -not authenticate digests or decide whether an edit was permitted. Parent quality -failures do not invalidate the edit lineage. Real generation-source dependencies -are inherited and must still be current; a child cannot drop them to escape an -invalidation. Rebase or composition is a new candidate with its own evidence. -This offline core predeclares dependency keys at initialization; introducing new -source authorities dynamically requires later controller integration. - -`register_attempt` binds a caller-owned attempt/execution key, reservation and -lease to one candidate, check and complete verification context. Registering an -attempt grants no execution permission. Unknown work on the same check cannot be -replaced, even after dependency invalidation. Valid outcomes cannot be resampled -until they pass; an INCONCLUSIVE outcome may be retried with a fresh identity -under the caller's frozen retry rules and a new reservation. - -`record_receipt` requires the entire registered binding. Identical delivery is -idempotent and conflicting delivery is rejected. History retains late results; -`evidence` admits only the latest attempt with current dependency epochs and -valid candidate provenance. FAIL is an observed outcome, whereas INCONCLUSIVE -leaves a check missing. Identical candidate bytes do not transfer evidence. - -`accept` selects only fully passing candidates, preferring the root on an -objective tie and otherwise using lexicographic candidate ID order. The result is -a **development selection**. Independent final/holdout acceptance and production -promotion remain caller-owned. `verified_gain` measures this selection relative -to the root. Invalidation recomputes eligibility; invalid root evidence blocks -acceptance and further drafting/check registration, while late archival remains -available. - -## Completion planning - -`completion_menu(state, jobs, available_budget=..., available_jobs=..., -max_targets=..., max_jobs=...)` takes a supplied integer-cost budget view and -returns immutable `Plan` values. The caller has already accounted for planning, -generation, final reserve, in-flight obligations and unknown spend when deriving -that view. Costs must be known positive conservative bounds in the same unit. - -Each `Job` names either a candidate/check pair or a shared preparation step. -Prerequisites form an acyclic graph. The solver takes each candidate's complete -missing-check set, closes its prerequisites, and enumerates unions of at most -`max_targets` candidate seeds. Every feasible singleton completion remains in -the menu. Shared jobs are charged once per plan, conflicts exclude combinations, -and available job slots and remaining check attempts bound each proposal. -`max_jobs` bounds the input graph; enumeration is intended for small offline -controls, not an unbounded production optimizer. No Scorer prediction is used -as evidence or as a substitute for a completion seed. - -Plans bind the search ID and state revision and carry a deterministic topological -job order. The caller retains the full job definitions and costs with its frozen -plan; job IDs alone do not authenticate them. The adapter rebuilds job -relations from current observations at each synchronous decision boundary; -already observed checks are not silently scheduled again. An in-flight attempt, -including an invalidated one, blocks replanning until its execution is settled. -An empty menu is a valid outcome under a tight budget; the root remains available -when its evidence is valid. Feasibility says nothing about whether checks pass. - -## Shared execution and persistence boundary - -These modules introduce no second runner, budget ledger, holdout tracker or -storage backend. A future application adapter must: - -1. Read current authoritative state and the frozen contract, settle the previous - work block, and supply a conservative available-budget view. -2. Generate and retain permitted edits, or select an affordable check plan using - the frozen roles. Account for generation, planning and failed work. -3. Use shared execution controls for atomic claim/reservation, absolute deadline, - resource isolation and actual execution. Bind their identities to the tree. -4. Authenticate and assess receipts, apply a transition against the expected - search revision, and persist it using the authoritative storage CAS/lock. - A pure function's revision check is **not a durable or cross-worker CAS**. - On conflict, reread and recompute; an ambiguous claim never grants a retry. -5. Persist real dependency invalidation before cancellation. Reconcile costs and - retain leases until process cleanup is confirmed. Receipt archival itself - neither charges again nor releases a resource. -6. Freeze the selected version for independent final acceptance, with holdout - access and interruptions tracked separately from development feedback. - -There is no production adapter in this slice. In particular, local immutable -values and revision checks do not establish crash recovery, external exactly-once, -multi-worker resource isolation, budget transactions or holdout non-reuse. - -## Verification and remaining delivery - -Run the CPU controls with: +# Speculative candidate search and group learning + +The experimental implementation of [issue #363](https://github.com/InferMatrixLab/InferRSI/issues/363) +composes bounded VATS, trainable CPU roles, GVO, durable work accounting and +independent final selection. It follows the two loops in +[SpecRSI v2 §3](https://arxiv.org/html/2610.16491v2#S3). It does not reproduce the +paper's language-model training or establish its reported gains. Existing +`campaign` and `benchmarks session` commands keep their serial lifecycle. + +## Supported profile and authority + +| Component | Implementation | Boundary | +| --- | --- | --- | +| Candidate/evidence model | `harness.search_tree`, `search_records` | Immutable artifacts, actual dependency epochs and complete-candidate selection | +| Solver | `harness.verification_plans`, `vats.menu` | Bounded exhaustive completion plans; conservative integer millisecond caps | +| Drafter | `model.search_roles`, `harness.search_adapter` | Trainable softmax over supplied exact text edits; actual retained changes, not free-form LLM generation | +| Policy | `model.search_roles` | Independent eight-feature linear softmax, recorded behavior probabilities, deterministic deployment actions | +| Scorer | `model.search_roles` | Separate linear model; actual suffix MSE and optional paid continuation differences | +| Work/state authority | `harness.work_control` | Transactions in the shared `RunStore` SQLite database | +| Execution | Existing `scenarios.external` and `local_process` | Absolute deadline and process-domain cleanup; no new subprocess runner | +| Engine | `apps.search_benchmarks.EngineEvaluator` | Existing packaged `optimization.py` as one whole check; currently the Qwen denoiser NPU task | +| Skill | `apps.search_benchmarks.SkillEvaluator`, checkout `tools.speculative_skill.Comparator` | Explicit provider configuration, original baseline, independent grader, separate development/holdout descriptors | +| Final | `harness.search_campaign.finalize` | One-time scope claim before access; receipt-only recovery/export; task promotion only | + +This is a trusted local host controller. Resource domains exclude **cooperating +controllers sharing one RunStore**. They are not scheduler/GPU reservations, +filesystem sandboxes, distributed leases, provider cancellation guarantees or +cryptographic attestations. The caller supplies authorized resources and frozen +edit/check permissions. Candidate code cannot enforce its own isolation. + +Execution uses one worker. The offline planner represents shared setup and +conflicts; supplied runtime adapters package setup into each whole check. They +do not execute arbitrary standalone shared-setup jobs. Multi-worker performance +qualification remains outstanding. + +## Candidate and evidence semantics + +`Contract` freezes search ID, context, roles, checks, objective/direction and +candidate/depth/attempt caps. Context includes plan, validator, workload, +environment, original baseline and reference. Objectives must be finite; +overflowing the difference between two finite scores is also rejected. Quality +checks are hard requirements, never compensable by a better score. + +`qualify_root` obtains root evidence through claimed real checks. Unknown root +acquisition can only recover its original receipt. `import_qualified_root` +explicitly re-scopes root receipts under identical artifact/check/context +conditions, retaining physical execution and reservation identities. Root check +attempts count against each search's cap; actual acquisition is charged once to +the campaign, separately from independent search returns. + +In `A → B → C`, B may be unchecked or fail quality while C is edited and checked. +C needs every required receipt for **C's own artifact** to win. Task failure does +not invalidate descendants. Genuine environment/source dependency invalidation +does invalidate dependent evidence. Dependencies are predeclared; dynamic +introduction of new authorities is unsupported. + +`record_receipt` archives registered late results; `evidence` admits only current, +matching receipts. Equal bytes alone do not authorize reuse. Duplicate delivery +is idempotent; conflicts fail. FAIL is an observation; INCONCLUSIVE leaves a +check missing. A valid outcome cannot be repeatedly sampled until it passes. +Permitted retries need new identities and paid reservations. + +`Search.invalidate` persists the epoch change and incumbent recomputation +atomically. An invalid root blocks search; late evidence can still be archived +and charged. `ACCEPT` prefers the root on ties, then candidate ID order. It +selects a development result without changing the stable version. + +## VATS execution + +`Search.start` retains a qualified root and frozen role/settings/adapter hashes. +Each decision claims and meters Solver/Scorer/Policy preparation, constructs +bounded DRAFT/VERIFY/ACCEPT actions and records the behavior law and paid state. +Completion bundles remain in the menu; Scorer predictions never become receipts. +Optional refinement adds unions without removing base completions. An undersized +menu cap fails explicitly. Collection samples Policy; deployment uses argmax. +Drafter edits remain sampled from the supplied permitted vocabulary. + +The chosen work block is persisted before execution. Each edit waits for its +parent artifact, without waiting for parent quality. Each check binds candidate, +contract, attempt, execution, reservation and domain. Settlement commits elapsed +time, cleanup facts and evidence/incumbent changes in one transaction. Recovery +continues the retained block cursor before making another Policy decision. + +ACCEPT and limits return only complete admissible versions. Unstarted work stops +when its conservative cap no longer fits. Unknown launched operations block +continuation until reconciled. Candidate, depth, step, menu and attempt limits +bound the search independently of its time budget. + +The planner closes prerequisites, counts shared costs once and checks conflicts. +`max_targets` limits completion **seeds**, not candidates incidentally completed +by cross-candidate prerequisites. `max_jobs` bounds the whole supplied job graph. +The exact algorithm targets small menus. `draft_ahead=False` and `bundled=False` +are explicit ablation switches; they are not claims of comparative efficiency. + +## GVO and paid continuations + +`collect_group` requires at least two independent complete searches with the same +root, contract, budget, snapshot and adapter. Each owns a tree and seed. The +campaign must be able to provide equal declared budgets before collection. +Leaves from one tree are not independent group members. + +`gvo.examples` computes whole-search returns and leave-one-out advantages. +Natural sampled Drafter/Policy choices supply the clipped PPO-style objective +with KL regularization against a frozen reference. Scorer targets use actual +remaining gains and future costs from each paid decision boundary. Current +preparation is sunk for that suffix but stays in total campaign cost. Missing +labels/cost observations are never zero-filled. Penalties and training settings +are explicit frozen conditions. + +`train_group` checks retained settled records and replayed behavior probabilities, +updates separate actual parameter arrays and commits checkpoint plus training +receipt. `learn` supports Reset/Carry and reuses completed updates on resume; +unknown training cannot be relaunched. An initial checkpoint with explicit +provenance is required. This profile provides neither a pretrained model nor a +supervised warmup dataset. + +`search_continuations.continue_from` copies a retained paid state, charges the +copy, forces one legal first action and continues the frozen policy. The entire +suffix is Scorer-only. `paired_targets` requires the same source state, settings, +snapshot, adapter and continuation seed with different first actions. Both +absolute suffix targets and observed relative differences train the Scorer. +Caller-authored unreceipted pair targets are rejected. + +These are real updates to a constrained CPU operator model. They are not LLM +finetuning, prompt summarization or evidence of pretrained-model improvement. + +## Accounting, recovery and final + +`WorkControl` extends the existing RunStore database with campaign, domain and +holdout tables. It is the authority for this experimental runtime, not a mirror +of serial-session manifests. One transaction reserves work/domains and registers +the pending tree attempt. Controllers using the same resource or holdout must +use the **same shared store**. + +Launch intent is durable before dispatch; an ambiguous crash never authorizes a +second launch. Known elapsed time and cleanup release unused reservation. +Unknown usage/cleanup retains the entire reservation and domain. Over-cap work +is charged and blocks promotion. Absolute deadlines include queueing and cleanup. +In-process bounded planning, editing and training are checked after return; +they are not forcibly preempted like subprocesses. Preflight freezing and initial +setup are outside this campaign meter, a remaining lifecycle-cost limitation. + +Receipts bind candidate/adapter identities and content hashes. They detect +accidental mutation under a trusted host, not malicious-host forgery. Recovery +checks retained candidate bytes and never launches new work. Lost preparation, +training or copy receipts remain unresolved; no guessed-cost retry is available. +External process-liveness reconciliation needs a qualified authority. + +`finalize` consumes a unique scope in the shared store before final data access, +freezes selection and closes search/training. Engine compares again with the +original baseline. For Skill, supply a **holdout-bound adapter** and the declared +holdout scope, never the development adapter. Exact bounded radix utility keeps +quality before token use; reduced tokens cannot offset a quality regression. + +After interruption, final consumes existing receipts only. Even an unstarted +remaining check cannot be newly dispatched by recovery. Existing final results +are immutable; export never reruns them. Only complete accepted final evidence, +known cleanup and valid budget can update `stable`. Role checkpoints are never +automatically promoted to a production/default improver. Such promotion needs +separate independent data and acceptance authority. + +Enforced spend is in milliseconds. Skill reports retain provider token/currency +facts, and the existing comparator enforces its per-comparison limits. A +**campaign-wide provider token/currency reservation bridge is not implemented**. +Elapsed-time caps must not be presented as that guarantee. + +## Entry points + +`inferrsi search engine --help` lists explicit Engine bounds and permissions. +Supply a fresh `--session`, shared `--store`, unique `--campaign`, packaged task, +initial checkpoint, edit vocabulary, authorized resource domain, time/search/ +training bounds, independent group count, rounds, final scope and final threshold. +Task protocol flags follow `--`, for example `--pairs 1 --samples 1 --warmups 0` +when the campaign owner chooses one measurement per arm. No gate is inferred. +A domain name does not allocate a device. + +Checkpoint JSON contains numeric arrays `drafter`, `policy`, `scorer` and nonempty +`origin`. Drafter dimensions match edits; Policy/Scorer each have eight dimensions. +Freeze initial parameters and provenance in the campaign plan. +`--edit TOKEN FILE BEFORE AFTER` permits one exact replacement in a candidate +relative file; candidates cannot change the evaluator or roles. + +Read/export without execution: + +```bash +inferrsi search status --session +inferrsi search export --session --output +``` + +Python composition uses `qualify_root`, `collect_group`, `learn`, `continue_from` +and `finalize`. Recovery requires the original frozen adapters. The CLI has no +blind whole-campaign restart command. Skill's checkout `Comparator` takes the +existing comparator configuration with an explicit bounded provider, original +Git baseline, editable bundle paths and two split descriptors (`path`, `sha256`, +`scope`, `task_count`). It uses the existing private Git-index materializer, +executes the existing comparator under the shared process executor and revalidates +its independent retained report. Construction does not load holdout bytes. +Provider readiness/credentials/authorization remain existing explicit requirements. + +## Validation and issue completion ```bash -python -m pytest tests/test_speculative_search.py tests/test_verification_plans.py +python -m pytest tests/test_speculative_search.py tests/test_verification_plans.py \ + tests/test_work_control.py tests/test_search_roles.py tests/test_vats_runtime.py \ + tests/test_search_benchmarks.py ``` -Tests cover retained file/directory artifacts for synthetic Engine/Skill shapes, -failed-parent repair, incomplete ancestors, failure versus missing evidence, -dependency/result order, identical bytes with separate attempts, stale revision -rejection, immutable role snapshots, deterministic selection, invalid roots and -bounded search. The planner is compared with an independent exhaustive job-subset -oracle across budgets and target limits, including shared preparation and conflicts. +Controls cover exhaustive planning, failed-parent repair, finite-score overflow, +concurrent domain claims, unknown reservations, atomic evidence/accounting, +invalid roots, process-backed CPU searches, numerical actor-gradient checks, +parameter updates, fresh searches, paid continuations, recovery and final export. +The numeric process grader and Skill report fixtures test software behavior; +they are not real task/provider efficacy evidence. -| Issue acceptance area | This slice | Remaining evidence | +| Issue area | Delivered software | Still required for full acceptance | | --- | --- | --- | -| AC-01/02 | Frozen scalar contract and offline candidate tree | Production support matrix and role/execution adapters | -| AC-03/05 | Scoped evidence, epoch invalidation, development selection and revision guards | Authoritative durable state integration and restart/race tests | -| AC-04 | Bounded exact completion menu and exhaustive controls | Scorer refinement, Policy selection and scalable solver integration | -| AC-06 | Frozen snapshot identity only | GVO data collection, objectives, parameter updates and new searches | -| AC-07/08 | References to caller-owned execution identities | Actual budget/resource transactions, cancellation and independent final/holdout | -| AC-09/10 | Synthetic control fixtures | Both real adapters, model generation, training, matched experiments and lifecycle costs | - -These tests do not establish model quality, engine speedup or SpecRSI efficiency. -The parent issue remains open for its full dual-loop delivery. +| AC-01/02 | Support/permission boundaries and immutable trees | Production role/environment qualification | +| AC-03/05 | Durable transitions, late receipts, invalidation, bounded synchronous VATS | Production adapter fault/race evidence | +| AC-04 | Exact planner, independent oracle and Scorer refinement | Scaling and real complementary-check evidence | +| AC-06 | Independent groups, actor/value updates, paid continuations, checkpoints, Reset/Carry | Actual task training results and role qualification | +| AC-07 | Shared local claims, elapsed-time caps and cleanup-aware recovery | Host callback preemption, provider/device total accounting and production resource integration | +| AC-08 | One-time final scope and task promotion gate | Real independent final evidence and separate default-improver qualification | +| AC-09 | Packaged Engine evaluator and explicit Skill comparator bridge | Both complete real VATS/GVO/final pilots | +| AC-10 | Retained trajectories/costs and ablation switches | Serial/VATS, bundle, learning, Reset/Carry and worker experiments with full lifecycle costs | + +The parent issue remains open. Software tests, CPU mathematical checks or an +available adapter do not establish full acceptance, NPU speedup, provider quality, +search efficiency or universal SpecRSI gains. diff --git a/src/inferrsi/apps/cli/__init__.py b/src/inferrsi/apps/cli/__init__.py index 974f3d9a..68f716a4 100644 --- a/src/inferrsi/apps/cli/__init__.py +++ b/src/inferrsi/apps/cli/__init__.py @@ -349,6 +349,11 @@ def main(argv: list[str] | None = None) -> int: result = getattr(agent_sessions, args.agent_action)(args.job) _print(result) return 0 + if args.command == "search": + from ..speculative_search import command as search_command + + _print(search_command(args)) + return 0 if args.command == "benchmarks": if args.benchmark_action == "session": from ...harness.optimization_session import session_command diff --git a/src/inferrsi/apps/cli/arguments.py b/src/inferrsi/apps/cli/arguments.py index 0b0c5da1..e8ec8f57 100644 --- a/src/inferrsi/apps/cli/arguments.py +++ b/src/inferrsi/apps/cli/arguments.py @@ -107,6 +107,9 @@ def parser() -> argparse.ArgumentParser: ) root.add_argument("--version", action="version", version=__version__) commands = root.add_subparsers(dest="command", required=True) + from ..speculative_search import add_arguments as search_arguments + + search_arguments(commands) mindie = commands.add_parser("mindie", help="Use project-approved, local MindIE knowledge") mindie_actions = mindie.add_subparsers(dest="mindie_action", required=True) enable = mindie_actions.add_parser("enable", help="Approve a MindIE domain for this checkout") diff --git a/src/inferrsi/apps/search_benchmarks.py b/src/inferrsi/apps/search_benchmarks.py new file mode 100644 index 00000000..eb7a1172 --- /dev/null +++ b/src/inferrsi/apps/search_benchmarks.py @@ -0,0 +1,159 @@ +"""Benchmark-owned evaluators composed with Harness speculative search. + +The engine protocol remains one indivisible check; no CPU or partial-case run +can impersonate its NPU performance qualification. Skill grading stays in the +existing independent comparator and quality-first rules. +""" + +import math +import sys +import time +from pathlib import Path + +from inferrsi.apps.improvement import assessment +from inferrsi.benchmark.improvement import SelectionPolicy, holdout_accepts +from inferrsi.benchmark.registry import task_resources +from inferrsi.harness import candidates, task_protocol +from inferrsi.harness.artifact_primitives import digest, sha256 +from inferrsi.harness.atomic_json import atomic_write_json +from inferrsi.harness.optimization_session import runtime_environment +from inferrsi.harness.scenarios.external import execute_trial + + +class EngineEvaluator: + def __init__(self, package, frozen, *, arguments, python=sys.executable, prefix=()): + self.frozen = Path(frozen).resolve() + self.plan = task_protocol.prepare( + Path(str(task_resources(package))), self.frozen, arguments + ) + self.python, self.prefix = python, tuple(prefix) + self.identity = digest( + { + "lane": "engine", + "task": package, + "plan": self.plan, + "python": python, + "prefix": self.prefix, + } + ) + + def evaluate(self, candidate, check, output, deadline): + if check != "evaluation": + raise ValueError("Packaged engine protocol requires one full evaluation job") + actual = { + p.relative_to(self.frozen).as_posix(): sha256(p) + for p in self.frozen.rglob("*") + if p.is_file() + } + if actual != self.plan["frozen_files"]: + raise ValueError("Frozen benchmark resources changed") + output.mkdir(parents=True) + candidates.retain(candidate / "candidate.py", output / "candidate.py") + protocol = task_protocol.protocol(self.frozen) + args = protocol.arguments( + output / "candidate.py", output / "measurement", self.plan["evaluation_settings"] + ) + from inferrsi.harness import optimization_session + + command = ( + *self.prefix, + self.python, + str(Path(optimization_session.__file__).resolve()), + str(self.frozen / "benchmark.py"), + *args, + ) + environment = runtime_environment() + environment["PYTHONDONTWRITEBYTECODE"] = "1" + execution = execute_trial( + command, + output, + output / "execution", + time.monotonic() + max(0, deadline - time.time()), + None, + environment=environment, + include_cleanup=True, + ) + assessed = protocol.assess(output, self.plan["evaluation_settings"], execution) + if execution["cleanup_verified"] is not True: + assessed = {"verdict": "INCONCLUSIVE", "score": None} + atomic_write_json(output / "assessment.json", assessed) + return { + "verdict": assessed["verdict"], + "objective": assessed.get("score"), + "cleanup_verified": execution["cleanup_verified"], + "assessment_sha256": sha256(output / "assessment.json"), + "execution_sha256": sha256(output / "execution/execution.json"), + } + + +class SkillEvaluator: + """Use an explicitly bound independent comparator, with an exact lexicographic utility. + + The comparator's run method returns a verified report and cleanup facts; it + must run the existing Skill comparison for the supplied candidate and split. + No generic model client or fixed-output fixture is silently selected here. + """ + + def __init__(self, comparator, *, split, policy, token_cap, task_count): + if not isinstance(policy, SelectionPolicy) or split not in ("development", "holdout"): + raise ValueError("A frozen Skill selection policy and explicit split are required") + if ( + type(token_cap) is not int + or token_cap < 1 + or type(task_count) is not int + or task_count < 1 + or task_count * (token_cap + 1) > 2**53 - 1 + ): + raise ValueError("Skill utility must be exactly representable within declared bounds") + self.comparator, self.split, self.policy = comparator, split, policy + self.holdout_scope = ( + getattr(comparator, "holdout_scope", None) if split == "holdout" else None + ) + self.token_cap, self.task_count = token_cap, task_count + self.identity = digest( + { + "lane": "skill", + "comparator": comparator.identity, + "split": split, + "min_success_rate": policy.min_success_rate, + "max_regressions": policy.max_regressions, + "token_cap": token_cap, + "task_count": task_count, + "utility": "successes*(token_cap+1)-tokens", + } + ) + + def evaluate(self, candidate, check, output, deadline): + if check != "evaluation": + raise ValueError("Skill comparison is an indivisible evaluation job") + report, cleanup = self.comparator.run(candidate, output, deadline, split=self.split) + score = assessment(report, self.policy) + tokens = score["usage"]["candidate"]["total_tokens"] + if ( + cleanup is not True + or not score["complete"] + or score["reason"] == "unknown_usage" + or tokens is None + or type(tokens) is not int + or not 0 <= tokens <= self.token_cap + or score["trials"] != self.task_count + ): + verdict, objective = "INCONCLUSIVE", None + elif score["reason"] not in ("eligible", "no_improvement"): + verdict, objective = "FAIL", None + else: + # One extra success dominates the entire possible token range. + # This exact radix preserves the existing quality-then-token order. + verdict = "PASS" + objective = score["successes"] * (self.token_cap + 1) - tokens + if not math.isfinite(objective): + raise ValueError("Invalid Skill utility") + if self.split == "holdout" and not holdout_accepts(score): + verdict, objective = "FAIL", None + return { + "verdict": verdict, + "objective": objective, + "cleanup_verified": cleanup, + "report_digest": digest(report), + "assessment": score, + } diff --git a/src/inferrsi/apps/speculative_search.py b/src/inferrsi/apps/speculative_search.py new file mode 100644 index 00000000..318939f0 --- /dev/null +++ b/src/inferrsi/apps/speculative_search.py @@ -0,0 +1,218 @@ +"""CLI composition for the supported packaged Engine VATS/GVO profile.""" + +import json +import sys +import time +from dataclasses import asdict, replace +from pathlib import Path + +from inferrsi.harness import candidates, search_tree +from inferrsi.harness.artifact_primitives import digest +from inferrsi.harness.atomic_json import atomic_write_json +from inferrsi.harness.run_store import RunStore, RunStoreReader +from inferrsi.harness.search_adapter import ArtifactAdapter, Edit +from inferrsi.harness.search_campaign import collect_group, finalize, learn, qualify_root +from inferrsi.harness.search_records import decode +from inferrsi.harness.vats import Settings +from inferrsi.harness.work_control import WorkControl +from inferrsi.model.search_roles import Snapshot + +from .search_benchmarks import EngineEvaluator + + +def add_arguments(commands): + search = commands.add_parser("search", help="Bounded SpecRSI search and role training") + actions = search.add_subparsers(dest="search_action", required=True) + run = actions.add_parser("engine", help="Run a supplied packaged Engine protocol") + run.add_argument("--session", type=Path, required=True) + run.add_argument("--store", type=Path, required=True, help="Shared resource/holdout RunStore") + run.add_argument("--campaign", required=True, help="Unique campaign ID in the shared store") + run.add_argument("--task", required=True) + run.add_argument( + "--edit", + nargs=4, + action="append", + required=True, + metavar=("TOKEN", "FILE", "BEFORE", "AFTER"), + ) + run.add_argument( + "--checkpoint", + type=Path, + required=True, + help="Initial trainable finite-edit role snapshot with provenance", + ) + run.add_argument("--domain", required=True, help="Caller-owned exclusive measurement domain") + run.add_argument("--max-seconds", type=int, required=True) + run.add_argument("--evaluation-seconds", type=int, required=True) + run.add_argument("--final-reserve-seconds", type=int, required=True) + run.add_argument("--search-seconds", type=int, required=True) + run.add_argument("--preparation-seconds", type=int, required=True) + run.add_argument("--draft-seconds", type=int, required=True) + run.add_argument("--training-seconds", type=int, required=True) + run.add_argument("--candidates", type=int, required=True) + run.add_argument("--depth", type=int, required=True) + run.add_argument("--attempts", type=int, required=True) + run.add_argument("--steps", type=int, required=True) + run.add_argument("--plans", type=int, required=True) + run.add_argument("--targets", type=int, required=True) + run.add_argument("--groups", type=int, required=True, help="Independent searches per round") + run.add_argument("--rounds", type=int, required=True) + run.add_argument("--learning-rate", type=float, required=True) + run.add_argument("--seed", type=int, default=0) + run.add_argument("--mode", choices=("reset", "carry"), default="reset") + run.add_argument("--final-scope", required=True) + run.add_argument("--minimum-final-score", type=float, required=True) + run.add_argument("--python", default=sys.executable) + run.add_argument("arguments", nargs="*", help="Task protocol arguments after --") + for action in ("status", "export"): + read = actions.add_parser(action) + read.add_argument("--session", type=Path, required=True) + if action == "export": + read.add_argument("--output", type=Path, required=True) + + +def command(args): + root = args.session.resolve() + if args.search_action != "engine": + location = json.loads((root / "control.json").read_text(encoding="utf-8")) + control = WorkControl(RunStoreReader(location["store"], "speculative")) + result = control.read(location["campaign"]) + if args.search_action == "export": + atomic_write_json(args.output, result) + return result + if root.exists(): + raise ValueError("Use a new session directory; status/export never restart work") + if args.groups < 2 or args.rounds < 1: + raise ValueError("GVO needs at least two independent searches and one training round") + if args.final_reserve_seconds < args.evaluation_seconds: + raise ValueError("Final reserve must cover the full evaluation cap") + required = ( + args.evaluation_seconds + + args.final_reserve_seconds + + (args.groups * args.rounds + 2) * args.search_seconds + + args.rounds * args.training_seconds + ) + if args.max_seconds < required: + raise ValueError( + "Campaign cap must fund qualification, all search groups, training and final" + ) + snapshot = Snapshot(**json.loads(args.checkpoint.read_text(encoding="utf-8"))) + if ( + len(snapshot.drafter) != len(args.edit) + or len(snapshot.policy) != 8 + or len(snapshot.scorer) != 8 + ): + raise ValueError( + "Checkpoint needs one Drafter weight per edit and eight Policy/Scorer weights" + ) + settings = Settings( + args.steps, + 1, + args.depth, + args.targets, + args.candidates, + args.plans, + args.preparation_seconds * 1000, + args.draft_seconds * 1000, + args.search_seconds * 1000, + ) + root.mkdir(parents=True) + evaluator = EngineEvaluator( + args.task, root / "frozen", arguments=args.arguments, python=args.python + ) + adapter = ArtifactAdapter( + root / "artifacts", + (Edit(*raw) for raw in args.edit), + {"evaluation": args.evaluation_seconds * 1000}, + evaluator, + domain=args.domain, + ) + source = root / "initial" + source.mkdir() + candidates.retain(root / "frozen/baseline.py", source / "candidate.py") + node = adapter.import_root(source, snapshot=digest(snapshot.document()), context={}) + control = WorkControl(RunStore(args.store.resolve(), "speculative")) + campaign = args.campaign + atomic_write_json( + root / "control.json", {"store": str(args.store.resolve()), "campaign": campaign} + ) + specification = { + "adapter": adapter.identity, + "settings": asdict(settings), + "groups": args.groups, + "rounds": args.rounds, + "mode": args.mode, + "learning_rate": args.learning_rate, + "minimum_final_score": args.minimum_final_score, + "initial_snapshot": snapshot.document(), + } + contract = search_tree.Contract( + "qualification", + search_tree.Context( + digest(specification), + digest(specification), + evaluator.identity, + args.domain, + args.task, + node.artifact_digest, + evaluator.identity, + ), + digest(snapshot.document()), + ("evaluation",), + "evaluation", + "maximize", + args.candidates, + args.depth, + args.attempts, + ) + control.create( + campaign, + specification, + {"searches": {}, "checkpoint": snapshot.document(), "stable": asdict(node)}, + cap_ms=args.max_seconds * 1000, + final_reserve_ms=args.final_reserve_seconds * 1000, + deadline=time.time() + args.max_seconds, + ) + initial = qualify_root(control, campaign, adapter, node, contract) + trained = learn( + control, + campaign, + adapter, + initial, + snapshot, + settings, + rounds=args.rounds, + group_size=args.groups, + training_cap_ms=args.training_seconds * 1000, + learning_rate=args.learning_rate, + seed=args.seed, + mode=args.mode, + ) + # A fresh post-update search demonstrates the new frozen checkpoint. Selection + # is development-only, and never uses final results to choose another version. + records = collect_group( + control, + campaign, + adapter, + initial, + trained, + settings, + group_id="deployment", + size=2, + seed=args.seed, + collect=False, + ) + states = [decode(record["tree"]) for record in records] + selected_state = max(states, key=search_tree.verified_gain) + selected = replace(search_tree.accept(selected_state), parent_id=None) + finalize( + control, + campaign, + adapter, + selected, + holdout_scope=args.final_scope, + accept_result=lambda values: all( + v["objective"] >= args.minimum_final_score for v in values + ), + ) + return control.read(campaign) diff --git a/src/inferrsi/harness/gvo.py b/src/inferrsi/harness/gvo.py new file mode 100644 index 00000000..10e95096 --- /dev/null +++ b/src/inferrsi/harness/gvo.py @@ -0,0 +1,213 @@ +"""GVO data separation and metered group updates of frozen search roles.""" + +import math +import time +import uuid +from dataclasses import replace + +from inferrsi.model.search_roles import Choice, finite, update + +from .artifact_primitives import digest +from .search_records import decode + + +def choice_from_document(raw): + return Choice( + **( + raw + | { + "options": tuple(raw["options"]), + "features": tuple(tuple(row) for row in raw["features"]), + "probabilities": tuple(raw["probabilities"]), + } + ) + ) + + +def examples(records, *, expense_weight=0.0, elapsed_weight=0.0): + records = tuple(records) + if len(records) < 2: + raise ValueError("GVO requires at least two independent complete searches") + if finite(expense_weight) < 0 or finite(elapsed_weight) < 0: + raise ValueError("Return penalties cannot be negative") + identities, roots, contracts, snapshots, settings, adapters = ( + [], + set(), + set(), + set(), + set(), + set(), + ) + returns = [] + for record in records: + if record["status"] != "complete" or record["pending"] is not None: + raise ValueError("Incomplete searches cannot supply GVO returns") + state = decode(record["tree"]) + identities.append(state.contract.search_id) + roots.add( + next(n.artifact_digest for n in state.candidates if n.candidate_id == state.root_id) + ) + frozen = record["tree"]["contract"].copy() + frozen.pop("search_id") + contracts.add(digest(frozen)) + snapshots.add(record["snapshot"]) + settings.add(record["settings"]) + adapters.add(record["adapter"]) + returns.append( + finite( + record["gain"] + - record["initial_gain"] + - expense_weight * finite(record["spent_ms"]) / 1000 + - elapsed_weight * finite(record["elapsed_ms"]) / 1000 + ) + ) + if len(set(identities)) != len(identities) or any( + len(values) != 1 for values in (roots, contracts, snapshots, settings, adapters) + ): + raise ValueError("Search group must have distinct IDs and identical starting conditions") + if any(record["forced"] for record in records): + raise ValueError("Forced continuations are Scorer-only, never actor group members") + advantages = [ + finite(value - math.fsum(v for j, v in enumerate(returns) if j != i) / (len(returns) - 1)) + for i, value in enumerate(returns) + ] + actors, values = [], [] + for record, advantage in zip(records, advantages): + values.extend( + suffix_examples(record, expense_weight=expense_weight, elapsed_weight=elapsed_weight) + ) + if not record["collect"]: + raise ValueError("Deployment searches cannot supply on-policy actor data") + for step in record["trace"]: + if step["forced"]: + raise ValueError("Forced action leaked into a natural trajectory") + for raw in (step["choice"], *step["draft_choices"]): + choice = choice_from_document(raw) + if choice.sampled: + actors.append((choice, advantage)) + return actors, values, {"search_ids": identities, "returns": returns, "advantages": advantages} + + +def suffix_examples(record, *, expense_weight, elapsed_weight): + if record["status"] != "complete" or record["pending"] is not None: + raise ValueError("Suffix target requires a completed and settled search") + end = record["started_at"] + record["elapsed_ms"] / 1000 + result = [] + for step in record["trace"]: + future_cost = record["spent_ms"] - step["spent_before"] + if future_cost < 0 or end < step["at"]: + raise ValueError("Suffix accounting precedes its paid decision state") + target = finite( + record["gain"] + - step["gain_before"] + - expense_weight * future_cost / 1000 + - elapsed_weight * (end - step["at"]) + ) + result.append((tuple(step["features"]), target)) + return result + + +def train_group( + control, + campaign, + records, + snapshot, + reference, + *, + cap_ms, + learning_rate, + expense_weight=0.0, + elapsed_weight=0.0, + scorer_continuations=(), + scorer_pairs=(), + pair_examples=(), + **training, +): + """One actual parameter update, durably checkpointed inside the campaign budget.""" + records = tuple(records) + if any(record["snapshot"] != digest(snapshot.document()) for record in records): + raise ValueError("Training checkpoint differs from collection checkpoint") + doc = control.read(campaign) + if doc["final_claim"] is not None: + raise ValueError("Training is closed before final evaluation") + if digest(doc["state"]["checkpoint"]) != digest(snapshot.document()): + raise ValueError("Training checkpoint is no longer current") + if any( + op["category"] == "training" and op["status"] != "settled" + for op in doc["operations"].values() + ): + raise ValueError("Unresolved training execution cannot be relaunched") + for record in (*records, *scorer_continuations, *(r for pair in scorer_pairs for r in pair)): + search_id = decode(record["tree"]).contract.search_id + retained = doc["state"]["searches"].get(search_id) + if retained is None or digest(retained) != digest(record): + raise ValueError("Training requires the retained, settled search record") + if pair_examples: + raise ValueError("Unreceipted pair targets are unsupported; use paid suffix records") + from .search_continuations import paired_targets + + pair_examples = tuple( + paired_targets(pair, expense_weight=expense_weight, elapsed_weight=elapsed_weight) + for pair in scorer_pairs + ) + operation = "gvo-" + uuid.uuid4().hex + control.claim( + campaign, + doc["revision"], + operation, + category="training", + cap_ms=cap_ms, + binding={ + "snapshot": digest(snapshot.document()), + "reference": digest(reference.document()), + "training": { + "learning_rate": learning_rate, + "expense_weight": expense_weight, + "elapsed_weight": elapsed_weight, + **training, + }, + "searches": [decode(r["tree"]).contract.search_id for r in records], + }, + ) + claim = control.launch(campaign, operation) + started = time.monotonic() + actors, values, summary = examples( + records, expense_weight=expense_weight, elapsed_weight=elapsed_weight + ) + for record in scorer_continuations: + if not record["forced"]: + raise ValueError("Additional Scorer trajectories must be explicitly marked forced") + values.extend( + suffix_examples(record, expense_weight=expense_weight, elapsed_weight=elapsed_weight) + ) + trained = update( + snapshot, + reference, + actors, + values, + pair_examples, + learning_rate=learning_rate, + groups=len(records), + **training, + ) + trained = replace(trained, origin=digest({"parent": snapshot.document(), "group": summary})) + elapsed = max(1, math.ceil((time.monotonic() - started) * 1000)) + + def checkpoint(state): + state["checkpoint"] = trained.document() + state.setdefault("training", []).append( + {**summary, "operation": operation, "checkpoint": digest(trained.document())} + ) + return state + + control.settle( + campaign, + operation, + {**summary, "checkpoint": trained.document()}, + actual_ms=elapsed, + cleanup_verified=True, + transition=checkpoint, + ) + if elapsed > cap_ms or time.time() >= claim["deadline"]: + raise ValueError("Training exceeded its frozen budget") + return trained diff --git a/src/inferrsi/harness/local_process.py b/src/inferrsi/harness/local_process.py index 1208e506..a12ad938 100644 --- a/src/inferrsi/harness/local_process.py +++ b/src/inferrsi/harness/local_process.py @@ -329,7 +329,7 @@ def _supervise( def execute_command(command: tuple[str, ...], cwd: Path, trial_dir: Path, deadline: float, *, - environment: dict | None = None) -> dict: + environment: dict | None = None, include_cleanup: bool = False) -> dict: """Legacy trial record: execute argv under a whole-run deadline in trial_dir.""" trial_dir.mkdir(parents=True) result_path = trial_dir / "result.json" @@ -358,4 +358,9 @@ def execute_command(command: tuple[str, ...], cwd: Path, trial_dir: Path, deadli if run["error"] is not None: record["error"] = run["error"] record["elapsed_seconds"] = run["elapsed_seconds"] + if include_cleanup: + record.update(cleanup_verified=(run["domain_empty"] if run["pid"] is not None else True), + termination_requested=run["termination_requested"], + process_exited=run["returncode"] is not None, + process_started_at=run["started_at"], process_ended_at=run["ended_at"]) return record diff --git a/src/inferrsi/harness/scenarios/external.py b/src/inferrsi/harness/scenarios/external.py index ba532614..b98826ad 100644 --- a/src/inferrsi/harness/scenarios/external.py +++ b/src/inferrsi/harness/scenarios/external.py @@ -17,6 +17,7 @@ def execute_trial( *, metric: Callable[[dict, str], float] | None = None, environment: dict | None = None, + include_cleanup: bool = False, ) -> dict: """Run one command and retain its record. @@ -27,6 +28,8 @@ def execute_trial( if metric_key is not None and metric is None: raise TypeError("execute_trial needs the caller's metric validation for metric_key") options = {"environment": environment} if environment is not None else {} + if include_cleanup: + options["include_cleanup"] = True record = execute_command(command, cwd, trial_dir, deadline, **options) result_path = Path(record["result_path"]) if metric_key is not None: diff --git a/src/inferrsi/harness/search_adapter.py b/src/inferrsi/harness/search_adapter.py new file mode 100644 index 00000000..4896f8a0 --- /dev/null +++ b/src/inferrsi/harness/search_adapter.py @@ -0,0 +1,283 @@ +"""Artifact-backed finite-edit generation and authenticated command evaluations. + +Commands and graders are trusted host inputs frozen before search. Candidates +must run in a sandbox supplied by the application when their code is untrusted; +the existing local process domain supplies cleanup, not filesystem isolation. +""" + +import json +import math +import time +from dataclasses import asdict, dataclass +from pathlib import Path + +from inferrsi.contracts.claim import Verdict +from inferrsi.model.search_roles import choose + +from . import candidates +from . import search_tree as tree +from .artifact_primitives import digest, sha256 +from .atomic_json import atomic_write_json +from .optimization_session import runtime_environment +from .scenarios.external import execute_trial +from .verification_plans import Job + + +@dataclass(frozen=True) +class Edit: + token: str + path: str + before: str + after: str + + def __post_init__(self): + path = Path(self.path) + if ( + not self.token + or not self.before + or self.before == self.after + or path.is_absolute() + or ".." in path.parts + or not path.parts + ): + raise ValueError("Edit must name one permitted relative file and a nonempty change") + + +class ArtifactAdapter: + def __init__(self, root, edits, checks, evaluator, *, domain): + self.root = Path(root).resolve() + self.root.mkdir(parents=True, exist_ok=True) + self.edits, self.checks = tuple(edits), dict(checks) + if ( + not self.edits + or len({e.token for e in self.edits}) != len(self.edits) + or not self.checks + or any(type(v) is not int or v <= 0 for v in self.checks.values()) + or not domain + ): + raise ValueError("Frozen edit vocabulary, check caps and domain are required") + self.evaluator, self.domain = evaluator, domain + self.identity = digest( + { + "edits": [asdict(e) for e in self.edits], + "checks": self.checks, + "evaluator": evaluator.identity, + "domain": domain, + "generation": "finite-edit-softmax-v1", + } + ) + + def path(self, node): + path = self.root / "candidates" / node.candidate_id + if path.parent != self.root / "candidates" or path.is_symlink(): + raise ValueError("Candidate ID must name one retained snapshot") + if candidates.identity(path) != node.artifact_digest: + raise ValueError("Retained candidate artifact changed") + return path + + def import_root(self, source, *, snapshot, context): + destination = self.root / "candidates" / "root" + destination.parent.mkdir(exist_ok=True) + fingerprint = candidates.retain(Path(source), destination) + return tree.Candidate( + "root", + fingerprint, + None, + tree.Producer(snapshot, self.identity, digest(context), digest([])), + ) + + def jobs(self, state): + return tuple( + Job(f"{node.candidate_id}:{check}", self.checks[check], node.candidate_id, check) + for node in state.candidates + if tree.candidate_valid(state, node.candidate_id) + and not any( + r.verdict == Verdict.FAIL for r in tree.evidence(state, node.candidate_id).values() + ) + for check in tree.missing_checks(state, node.candidate_id) + ) + + def domains(self, kind, node, check): + return (self.domain,) if kind == "verify" else () + + def _save(self, operation, binding, result): + if Path(operation).name != operation or operation in (".", ".."): + raise ValueError("Operation must name one retained execution") + binding = binding | {"adapter": self.identity} + path = self.root / "receipts" / (operation + ".json") + path.parent.mkdir(exist_ok=True) + atomic_write_json( + path, + { + "binding": binding, + "result": result, + "digest": digest({"binding": binding, "result": result}), + }, + ) + return result + + def draft(self, parent, state, snapshot, rng, operation, deadline): + started = time.monotonic() + source = self.path(parent) + files = candidates.members(source) + legal = [ + (index, edit) + for index, edit in enumerate(self.edits) + if edit.path in files + and files[edit.path].read_text(encoding="utf-8").count(edit.before) == 1 + ] + choice, node = None, None + if legal and time.time() < deadline: + vectors = [ + tuple(float(i == index) for i in range(len(self.edits))) for index, _ in legal + ] + choice = choose( + snapshot, + "drafter", + [edit.token for _, edit in legal], + vectors, + rng=rng, + context=digest(asdict(parent)), + collect=True, + ) + edit = legal[choice.selected][1] + destination = self.root / "candidates" / operation + candidates.retain(source, destination) + path = destination / edit.path + text = path.read_text(encoding="utf-8") + path.write_text(text.replace(edit.before, edit.after, 1), encoding="utf-8") + node = tree.Candidate( + operation, + candidates.identity(destination), + parent.candidate_id, + tree.Producer( + digest(snapshot.document()), + self.identity, + digest(asdict(parent)), + digest([asdict(r) for r in state.receipts]), + ), + parent.dependencies, + ) + result = { + "candidate": asdict(node) if node else None, + "choice": asdict(choice) if choice else None, + "cleanup_verified": True, + "elapsed_ms": max(1, math.ceil((time.monotonic() - started) * 1000)), + } + if time.time() >= deadline: + result["candidate"] = None + result["error"] = "draft_deadline" + return self._save(operation, {"candidate": asdict(parent), "kind": "draft"}, result) + + def verify(self, node, check, operation, deadline): + started = time.monotonic() + source = self.path(node) + output = self.root / "executions" / operation + result = self.evaluator.evaluate(source, check, output, deadline) + if candidates.identity(source) != node.artifact_digest: + raise ValueError("Evaluation changed the retained candidate") + verdict = Verdict(result["verdict"]) + objective = result.get("objective") + if objective is not None and ( + type(objective) not in (int, float) or not math.isfinite(objective) + ): + raise ValueError("Nonfinite benchmark score") + result = { + **result, + "verdict": verdict.value, + "elapsed_ms": max(1, math.ceil((time.monotonic() - started) * 1000)), + "evidence_digest": digest(result), + "objective": objective, + } + return self._save( + operation, {"candidate": asdict(node), "check": check, "kind": "verify"}, result + ) + + def recover(self, operation, pending): + if Path(operation).name != operation or operation in (".", ".."): + raise ValueError("Operation must name one retained execution") + path = self.root / "receipts" / (operation + ".json") + if not path.is_file(): + return None + receipt = json.loads(path.read_text(encoding="utf-8")) + if ( + digest({"binding": receipt["binding"], "result": receipt["result"]}) + != receipt["digest"] + ): + raise ValueError("Recovery receipt changed") + expected = { + "candidate": pending["binding"]["candidate"], + "kind": pending["kind"], + "adapter": self.identity, + } + if pending["kind"] == "verify": + expected["check"] = pending["binding"]["check"] + if digest(expected) != digest(receipt["binding"]): + raise ValueError("Recovery receipt belongs to a different operation") + for raw in (expected["candidate"], receipt["result"].get("candidate")): + if raw is not None: + self.path( + tree.Candidate( + **( + raw + | { + "producer": tree.Producer(**raw["producer"]), + "dependencies": tuple( + tree.Dependency(**d) for d in raw["dependencies"] + ), + } + ) + ) + ) + return receipt["result"] + + +class CommandEvaluator: + """Frozen independent evaluator command with explicit JSON result protocol. + + The grader writes {verdict, objective?} to {result}; the shared executor + independently supplies process/cleanup facts. Useful for project protocols + and CPU integration controls, not an implicit production qualification. + """ + + def __init__(self, commands, resources): + self.commands = {key: tuple(value) for key, value in commands.items()} + self.resources = {Path(path).resolve(): sha256(Path(path)) for path in resources} + self.identity = digest( + {"commands": self.commands, "resources": {str(k): v for k, v in self.resources.items()}} + ) + + def evaluate(self, candidate, check, output, deadline): + if any(sha256(path) != expected for path, expected in self.resources.items()): + raise ValueError("Frozen evaluator resources changed") + command = tuple(arg.replace("{candidate}", str(candidate)) for arg in self.commands[check]) + output.parent.mkdir(parents=True, exist_ok=True) + record = execute_trial( + command, + candidate, + output, + time.monotonic() + max(0, deadline - time.time()), + None, + environment=runtime_environment(), + include_cleanup=True, + ) + result = { + "verdict": "INCONCLUSIVE", + "objective": None, + "cleanup_verified": record["cleanup_verified"], + "execution_sha256": sha256(output / "execution.json"), + } + if ( + record["status"] == "completed" + and record["returncode"] == 0 + and record["cleanup_verified"] is True + ): + document = json.loads(Path(record["result_path"]).read_text(encoding="utf-8")) + result.update( + verdict=Verdict(document["verdict"]).value, + objective=document.get("objective"), + result_sha256=sha256(Path(record["result_path"])), + ) + if any(sha256(path) != expected for path, expected in self.resources.items()): + raise ValueError("Evaluator resources changed during execution") + return result diff --git a/src/inferrsi/harness/search_campaign.py b/src/inferrsi/harness/search_campaign.py new file mode 100644 index 00000000..fea8bce9 --- /dev/null +++ b/src/inferrsi/harness/search_campaign.py @@ -0,0 +1,341 @@ +"""Campaign assembly: real root acquisition, independent searches, GVO and final gating.""" + +import time +from dataclasses import asdict, replace + +from inferrsi.contracts.claim import Verdict +from inferrsi.model.search_roles import Snapshot + +from . import search_tree as tree +from .artifact_primitives import digest +from .gvo import train_group +from .search_records import decode, encode +from .vats import Search + + +def qualify_root(control, campaign, adapter, root, contract): + """Acquire root evidence using real checks and the same authoritative accounting.""" + retained = control.read(campaign)["state"].get("qualified_root") + if retained is not None: + state = decode(retained) + if state.contract != contract or tree.candidate(state, state.root_id) != root: + raise ValueError("Root qualification is bound to another frozen context") + tree.accept(state) + return state + receipts = [] + for check in contract.checks: + doc = control.read(campaign) + binding = { + "candidate": asdict(root), + "context": asdict(contract.context), + "check": check, + "adapter": adapter.identity, + } + operation = "root-" + digest(binding) + previous = doc["operations"].get(operation) + if previous is None: + control.claim( + campaign, + doc["revision"], + operation, + category="root", + cap_ms=adapter.checks[check], + domains=(adapter.domain,), + binding=binding, + ) + claim = control.launch(campaign, operation) + result = adapter.verify(root, check, operation, claim["deadline"]) + elif previous["status"] == "settled": + result = previous["receipt"] + else: + result = adapter.recover(operation, {"kind": "verify", "binding": binding}) + if result is None: + raise ValueError("Root execution unknown; recover its receipt without relaunch") + control.settle( + campaign, + operation, + result, + actual_ms=result.get("elapsed_ms"), + cleanup_verified=result.get("cleanup_verified"), + ) + if ( + result["cleanup_verified"] is not True + or result["verdict"] != "PASS" + or control.read(campaign)["blocked"] + ): + raise ValueError("Root lacks complete admissible real evidence") + attempt = tree.Attempt( + operation, + operation, + operation, + operation, + contract.search_id, + root.candidate_id, + root.artifact_digest, + check, + contract.context, + root.dependencies, + ) + receipts.append( + tree.Receipt(attempt, Verdict.PASS, result["evidence_digest"], result.get("objective")) + ) + state = tree.initialize(contract, root, receipts, dependencies=root.dependencies) + doc = control.read(campaign) + control.update(campaign, doc["revision"], lambda body: body | {"qualified_root": encode(state)}) + return state + + +def import_qualified_root(source, contract): + """Explicit root-only evidence import under identical artifact/check/context scope. + + No new physical execution is asserted: execution/reservation/lease keys and + evidence digests retain their original identity. The receiving attempt is a + scoped import recorded in the campaign; arbitrary child evidence is not reused. + """ + if ( + source.contract.context != contract.context + or source.contract.checks != contract.checks + or source.contract.objective_check != contract.objective_check + or source.contract.direction != contract.direction + ): + raise ValueError("Root evidence reuse requires an identical verification contract") + root = tree.candidate(source, source.root_id) + observed = tree.evidence(source, source.root_id) + if len(observed) != len(contract.checks) or any( + r.verdict != Verdict.PASS for r in observed.values() + ): + raise ValueError("Cannot import invalid root evidence") + receipts = [ + replace( + record, + attempt=replace( + record.attempt, + search_id=contract.search_id, + attempt_id=f"{contract.search_id}-import-{check}", + ), + ) + for check, record in observed.items() + ] + return tree.initialize(contract, root, receipts, dependencies=source.dependencies) + + +def collect_group( + control, campaign, adapter, root, snapshot, settings, *, group_id, size, seed=0, collect=True +): + if type(size) is not int or size < 2: + raise ValueError("A group needs two or more independent searches") + doc = control.read(campaign) + outstanding = sum( + max( + 0, + settings.search_ms + - doc["state"]["searches"].get(f"{group_id}-{i}", {}).get("spent_ms", 0), + ) + for i in range(size) + if doc["state"]["searches"].get(f"{group_id}-{i}", {}).get("status") != "complete" + ) + if outstanding > min(control.available(doc), (doc["deadline"] - time.time()) * 1000): + raise ValueError("Group cannot provide identical search budgets within the campaign cap") + result = [] + for index in range(size): + search_id = f"{group_id}-{index}" + contract = replace( + root.contract, search_id=search_id, improver_snapshot=digest(snapshot.document()) + ) + initial = import_qualified_root(root, contract) + search = Search(control, campaign, search_id, snapshot, settings, adapter) + doc = control.read(campaign) + if search_id not in doc["state"]["searches"]: + search.start(initial, seed=seed + index, collect=collect) + else: + _, retained, retained_tree = search.read() + if ( + retained["seed"] != seed + index + or retained["collect"] != collect + or retained_tree.contract != contract + or tree.candidate(retained_tree, retained_tree.root_id) + != tree.candidate(initial, initial.root_id) + ): + raise ValueError("Resumed group has different frozen starting conditions") + result.append(search.run()) + return tuple(result) + + +def learn( + control, + campaign, + adapter, + root, + snapshot, + settings, + *, + rounds, + group_size, + training_cap_ms, + learning_rate, + seed=0, + mode="reset", + **training, +): + if type(rounds) is not int or rounds < 1 or mode not in ("reset", "carry"): + raise ValueError("Frozen positive outer rounds and Reset/Carry mode are required") + reference = snapshot + for index in range(rounds): + records = collect_group( + control, + campaign, + adapter, + root, + snapshot, + settings, + group_id=f"round-{index}", + size=group_size, + seed=seed + index * group_size, + ) + ids = [decode(r["tree"]).contract.search_id for r in records] + doc = control.read(campaign) + retained = [ + op + for op in doc["operations"].values() + if op["category"] == "training" and op["binding"]["searches"] == ids + ] + if retained: + if len(retained) != 1 or retained[0]["status"] != "settled": + raise ValueError("Unknown training outcome; no second update is permitted") + op = retained[0] + expected = { + "learning_rate": learning_rate, + "expense_weight": 0.0, + "elapsed_weight": 0.0, + **training, + } + if ( + op["binding"]["snapshot"] != digest(snapshot.document()) + or op["binding"]["reference"] != digest(reference.document()) + or digest(op["binding"]["training"]) != digest(expected) + ): + raise ValueError("Frozen outer-loop training configuration changed") + snapshot = Snapshot(**op["receipt"]["checkpoint"]) + else: + snapshot = train_group( + control, + campaign, + records, + snapshot, + reference, + cap_ms=training_cap_ms, + learning_rate=learning_rate, + **training, + ) + if mode == "carry": + best = min( + (decode(record["tree"]) for record in records), + key=lambda state: (-tree.verified_gain(state), state.contract.search_id), + ) + selected = tree.accept(best) + # Carry the complete selected artifact, retaining its actual receipts. + selected = replace(selected, parent_id=None) + root = tree.initialize( + best.contract, + selected, + tree.evidence(best, selected.candidate_id).values(), + dependencies=best.dependencies, + ) + return snapshot + + +def finalize(control, campaign, adapter, selected, *, holdout_scope, accept_result): + """Freeze once; a failed/unknown holdout never reopens search or changes selection. + + accept_result is a fixed independent final gate. Only this function records + stable promotion; search acceptance and trained checkpoints do not promote. + """ + doc = control.read(campaign) + if doc["final_result"] is not None: + return doc["final_result"] + required_scope = getattr(adapter.evaluator, "holdout_scope", None) + if required_scope is not None and holdout_scope != required_scope: + raise ValueError("Final scope must match the evaluator's frozen holdout data identity") + binding = { + "candidate": asdict(selected), + "adapter": adapter.identity, + "checks": adapter.checks, + "scope": holdout_scope, + } + recovering = doc["final_claim"] is not None + if recovering: + if digest(doc["final_claim"]["binding"]) != digest(binding): + raise ValueError("Cannot change the final candidate or validation scope") + else: + control.claim_final(campaign, doc["revision"], holdout_scope, binding) + results = [] + for check, cap in adapter.checks.items(): + doc = control.read(campaign) + operation = "final-" + digest({"campaign": campaign, "binding": binding, "check": check}) + if recovering: + op = doc["operations"].get(operation) + if op is None: + raise ValueError("Interrupted holdout may be consumed; no new execution") + if op["status"] == "settled": + result = op["receipt"] + else: + result = adapter.recover( + operation, + {"kind": "verify", "binding": {"candidate": asdict(selected), "check": check}}, + ) + if result is None: + raise ValueError("Unknown final execution; recover its receipt without rerun") + control.settle( + campaign, + operation, + result, + actual_ms=result.get("elapsed_ms"), + cleanup_verified=result.get("cleanup_verified"), + ) + else: + control.claim( + campaign, + doc["revision"], + operation, + category="final", + cap_ms=cap, + domains=(adapter.domain,), + binding=binding | {"check": check}, + final=True, + ) + claim = control.launch(campaign, operation) + result = adapter.verify(selected, check, operation, claim["deadline"]) + control.settle( + campaign, + operation, + result, + actual_ms=result.get("elapsed_ms"), + cleanup_verified=result.get("cleanup_verified"), + ) + results.append(result) + doc = control.read(campaign) + unresolved = doc["blocked"] or any( + op["status"] != "settled" for op in doc["operations"].values() + ) + passed = ( + not unresolved + and all(r["verdict"] == "PASS" and r["cleanup_verified"] is True for r in results) + and accept_result(results) + ) + result = { + "candidate": selected.candidate_id, + "verdict": "PASS" if passed else "FAIL", + "results": results, + "promoted": passed, + } + if unresolved or any( + r["verdict"] == "INCONCLUSIVE" or r["cleanup_verified"] is not True for r in results + ): + result.update(verdict="INCONCLUSIVE", promoted=False) + control.finish_final( + campaign, + binding, + result, + transition=(lambda body: body | {"stable": asdict(selected)}) if passed else None, + ) + return result diff --git a/src/inferrsi/harness/search_continuations.py b/src/inferrsi/harness/search_continuations.py new file mode 100644 index 00000000..665243f7 --- /dev/null +++ b/src/inferrsi/harness/search_continuations.py @@ -0,0 +1,155 @@ +"""Paid-state continuation sampling, isolated from natural GVO actor groups.""" + +import copy +import math +import time +from dataclasses import asdict, replace + +from .artifact_primitives import digest +from .search_records import decode, encode +from .vats import Action, Search + + +def continue_from(search, step_index, action_key, search_id, *, copy_cap_ms, seed): + """Clone a retained paid decision state and force exactly its first action. + + Original observations are explicit imports with their physical execution IDs. + Copying is charged; all suffix edits/checks use ordinary WorkControl claims. + No holdout or unobserved outcome is used. The entire suffix is Scorer-only. + """ + doc, source, _ = search.read() + if doc["final_claim"] is not None or source["status"] != "complete": + raise ValueError("Continuation source must be a completed development search") + if ( + source["forced"] + or type(step_index) is not int + or not 0 <= step_index < len(source["trace"]) + ): + raise ValueError("Select a retained natural paid decision state") + step = source["trace"][step_index] + actions = [Action(**raw) for raw in step["menu"]] + selected = next((i for i, action in enumerate(actions) if action.key == action_key), None) + if selected is None: + raise ValueError("Forced action is outside the original paid menu") + action = actions[selected] + fork = Search( + search.control, search.campaign, search_id, search.snapshot, search.settings, search.adapter + ) + if search_id in doc["state"]["searches"]: + _, retained, _ = fork.read() + if retained.get("continuation") != { + "source": search.search_id, + "step": step_index, + "action": action_key, + "paid_state": digest(step["paid_state"]), + "seed": seed, + }: + raise ValueError("Continuation identity already belongs to another paid state") + return fork.run() + operation = "copy-" + search_id + search.control.claim( + search.campaign, + doc["revision"], + operation, + category="continuation_copy", + cap_ms=copy_cap_ms, + binding={"search": search_id, "source": search.search_id, "step": step_index}, + ) + search.control.launch(search.campaign, operation) + started = time.monotonic() + current = decode(copy.deepcopy(step["paid_state"])) + # Change the receiving scope, retaining immutable physical source identities. + imports = { + attempt.attempt_id: replace( + attempt, search_id=search_id, attempt_id=f"{search_id}-import-{attempt.attempt_id}" + ) + for attempt in current.attempts + } + current = replace( + current, + contract=replace(current.contract, search_id=search_id), + attempts=tuple(imports[a.attempt_id] for a in current.attempts), + receipts=tuple(replace(r, attempt=imports[r.attempt.attempt_id]) for r in current.receipts), + ) + record = copy.deepcopy(source) + forced_step = copy.deepcopy(step) + now = time.time() + forced_step.update( + action=asdict(action), + features=step["vectors"][selected], + forced=True, + draft_choices=[], + at=now, + ) + record.update( + tree=encode(current), + seed=seed, + forced=True, + trace=[forced_step], + status="active", + pending=None, + spent_ms=step["spent_before"], + started_at=now, + elapsed_ms=0, + draft_attempts=step["draft_attempts"], + initial_gain=step["gain_before"], + initial_spent_ms=step["spent_before"], + continuation={ + "source": search.search_id, + "step": step_index, + "action": action_key, + "paid_state": digest(step["paid_state"]), + "seed": seed, + }, + active_action=None + if action.kind == "accept" + else { + "action": asdict(action), + "cursor": 0, + "last_child": None, + "jobs": step["jobs"], + }, + ) + record.pop("gain", None) + record.pop("stop_reason", None) + elapsed = max(1, math.ceil((time.monotonic() - started) * 1000)) + + def commit(state): + if search_id in state["searches"]: + raise ValueError("Continuation search was concurrently created") + state["searches"][search_id] = record + return state + + search.control.settle( + search.campaign, + operation, + {"record": record}, + actual_ms=elapsed, + cleanup_verified=True, + transition=commit, + ) + if action.kind == "accept": + return fork.finish("forced_accept") + return fork.run() + + +def paired_targets(records, *, expense_weight, elapsed_weight): + """Produce differences only for two settled continuations of the same paid state.""" + from .gvo import suffix_examples + + if len(records) != 2: + raise ValueError("A matched comparison needs exactly two continuations") + left, right = records + keys = ("source", "step", "paid_state", "seed") + if ( + not all(r["forced"] and r.get("continuation") for r in records) + or any(left["continuation"][k] != right["continuation"][k] for k in keys) + or any(left[k] != right[k] for k in ("snapshot", "settings", "adapter")) + or left["continuation"]["action"] == right["continuation"]["action"] + ): + raise ValueError("Scorer pair must share a paid state and frozen continuation policy") + first = [ + suffix_examples(r, expense_weight=expense_weight, elapsed_weight=elapsed_weight)[0] + for r in records + ] + return (first[0][0], first[1][0], first[0][1] - first[1][1]) diff --git a/src/inferrsi/harness/search_records.py b/src/inferrsi/harness/search_records.py new file mode 100644 index 00000000..8377a358 --- /dev/null +++ b/src/inferrsi/harness/search_records.py @@ -0,0 +1,60 @@ +"""Explicit JSON encoding of immutable search controls, without executable payloads.""" + +from dataclasses import asdict + +from inferrsi.contracts.claim import Verdict + +from . import search_tree as tree + + +def encode(state): + return asdict(state) + + +def decode(raw): + def dependencies(values): + return tuple(tree.Dependency(**value) for value in values) + + def attempt(value): + return tree.Attempt( + **( + value + | { + "context": tree.Context(**value["context"]), + "dependencies": dependencies(value["dependencies"]), + } + ) + ) + + contract = tree.Contract( + **(raw["contract"] | {"context": tree.Context(**raw["contract"]["context"])}) + ) + nodes = tuple( + tree.Candidate( + **( + value + | { + "producer": tree.Producer(**value["producer"]), + "dependencies": dependencies(value["dependencies"]), + } + ) + ) + for value in raw["candidates"] + ) + return tree.SearchState( + contract, + raw["root_id"], + nodes, + tuple(attempt(value) for value in raw["attempts"]), + tuple( + tree.Receipt( + attempt(value["attempt"]), + Verdict(value["verdict"]), + value["evidence_digest"], + value["objective"], + ) + for value in raw["receipts"] + ), + dependencies(raw["dependencies"]), + raw["revision"], + ) diff --git a/src/inferrsi/harness/search_tree.py b/src/inferrsi/harness/search_tree.py index d6623835..e2fa8fe4 100644 --- a/src/inferrsi/harness/search_tree.py +++ b/src/inferrsi/harness/search_tree.py @@ -256,6 +256,8 @@ def verified_gain(state): check = state.contract.objective_check gain = (evidence(state, selected.candidate_id)[check].objective - evidence(state, state.root_id)[check].objective) + if not isfinite(gain): + raise ValueError("Verified gain exceeds the finite objective domain") return gain if state.contract.direction == "maximize" else -gain diff --git a/src/inferrsi/harness/vats.py b/src/inferrsi/harness/vats.py new file mode 100644 index 00000000..bf79022f --- /dev/null +++ b/src/inferrsi/harness/vats.py @@ -0,0 +1,659 @@ +"""Bounded synchronous VATS over immutable candidates and injected real operations. + +The controller owns search transitions; WorkControl owns persistent claims and +accounting; the adapter owns permitted edits and authenticated real observations. +No value prediction is admitted as evidence. There is no background re-dispatch. +""" + +import math +import random +import time +import uuid +from dataclasses import asdict, dataclass, replace +from itertools import combinations + +from inferrsi.model.search_roles import choose, dot + +from . import search_tree as tree +from .artifact_primitives import digest +from .search_records import decode, encode +from .verification_plans import completion_menu + + +@dataclass(frozen=True) +class Settings: + max_steps: int + max_parents: int + draft_depth: int + max_targets: int + max_jobs: int + max_plans: int + preparation_ms: int + draft_ms: int + search_ms: int + refine_queries: int = 0 + temperature: float = 1.0 + exploration: float = 0.1 + draft_ahead: bool = True + bundled: bool = True + + def __post_init__(self): + for key in ( + "max_steps", + "max_parents", + "draft_depth", + "max_targets", + "max_jobs", + "max_plans", + "preparation_ms", + "draft_ms", + "search_ms", + ): + if type(getattr(self, key)) is not int or getattr(self, key) <= 0: + raise ValueError("VATS bounds must be positive integers") + if type(self.refine_queries) is not int or self.refine_queries < 0: + raise ValueError("Invalid Scorer query cap") + if not math.isfinite(self.temperature) or self.temperature <= 0: + raise ValueError("Invalid policy temperature") + if not 0 <= self.exploration < 1: + raise ValueError("Invalid policy exploration") + + +@dataclass(frozen=True) +class Action: + kind: str + parents: tuple[str, ...] = () + depth: int = 0 + jobs: tuple[str, ...] = () + cost_ms: int = 0 + + @property + def key(self): + return digest(asdict(self)) + + +def depth(state, node): + result = 0 + while node.parent_id is not None: + result += 1 + node = tree.candidate(state, node.parent_id) + return result + + +def features(state, action, available, settings): + """Fixed public features; ACCEPT has a fixed zero Policy logit.""" + if action.kind == "accept": + return (0.0,) * 8 + return ( + 1.0, + float(action.kind == "draft"), + float(action.kind == "verify"), + action.cost_ms / settings.search_ms, + len(state.candidates) / state.contract.max_candidates, + available / settings.search_ms, + action.depth / state.contract.max_depth, + len(action.jobs) / settings.max_jobs, + ) + + +def menu(state, jobs, available, settings, snapshot): + """Preserve singleton completion seeds before optionally refining their unions.""" + available = min(available, settings.search_ms) + plans = completion_menu( + state, + jobs, + available_budget=max(0, available), + available_jobs=settings.max_jobs, + max_targets=settings.max_targets, + max_jobs=settings.max_jobs, + ) + singleton_plans = completion_menu( + state, + jobs, + available_budget=max(0, available), + available_jobs=settings.max_jobs, + max_targets=1, + max_jobs=settings.max_jobs, + ) + seeds = [Action("verify", jobs=plan.job_ids, cost_ms=plan.cost) for plan in singleton_plans] + unions = [Action("verify", jobs=plan.job_ids, cost_ms=plan.cost) for plan in plans] + allowed = {frozenset(a.jobs): a for a in unions} + # Refine priority among feasible bounded unions without evicting completion + # seeds or bypassing Solver's frozen target/job/cost/attempt constraints. + by_job = {job.job_id: job for job in jobs} + queries = 0 + extra = [] + for left, right in combinations(seeds, 2): + if queries + 2 > settings.refine_queries: + break + union = tuple(dict.fromkeys((*left.jobs, *right.jobs))) + if frozenset(union) not in allowed: + continue + cost = sum(by_job[jid].cost for jid in union) + attempts = sum(by_job[jid].candidate_id is not None for jid in union) + if ( + cost > available + or len(union) > settings.max_jobs + or attempts + len(state.attempts) > state.contract.max_attempts + or any(set(by_job[jid].conflicts) & set(union) for jid in union) + ): + continue + trial = allowed[frozenset(union)] + queries += 2 + if dot(snapshot.scorer, features(state, trial, available, settings)) > dot( + snapshot.scorer, features(state, left, available, settings) + ): + extra.append(trial) + if not settings.bundled: + # Ablation: individual executable jobs, explicitly retaining incomplete evidence. + seeds = [ + Action("verify", jobs=(job.job_id,), cost_ms=job.cost) + for job in jobs + if not job.prerequisites + and job.cost <= available + and job.candidate_id is not None + and job.check_id in tree.missing_checks(state, job.candidate_id) + ] + extra = [] + unions = [] + parents = [ + node + for node in state.candidates + if tree.candidate_valid(state, node.candidate_id) + and (settings.draft_ahead or node in tree.complete_candidates(state)) + ] + drafts = [] + slots = state.contract.max_candidates - len(state.candidates) + for count in range(1, min(len(parents), settings.max_parents) + 1): + for selected in combinations(parents, count): + for height in range(1, settings.draft_depth + 1): + cost = count * height * settings.draft_ms + if ( + cost <= available + and count * height <= slots + and all( + depth(state, node) + height <= state.contract.max_depth for node in selected + ) + ): + drafts.append( + Action( + "draft", + tuple(node.candidate_id for node in selected), + height, + cost_ms=cost, + ) + ) + if len(drafts) >= settings.max_plans: + break + if len(drafts) >= settings.max_plans: + break + if len(drafts) >= settings.max_plans: + break + required = [Action("accept"), *seeds] + if len(required) > settings.max_plans: + raise ValueError("Menu cap cannot preserve all feasible completion seeds") + unique = {action.key: action for action in required} + for action in (*extra, *drafts, *unions): + if len(unique) >= settings.max_plans: + break + unique[action.key] = action + return tuple(unique.values()) + + +class Search: + """A resumable search bound to one immutable role snapshot and adapter contract. + + Adapter methods: jobs(tree), draft(parent, choice_context, snapshot, rng, + destination, deadline), verify(node, check, destination, deadline), recover(...). + Draft returns (Candidate, Choice|None, receipt); verify returns Receipt facts. + Both return authenticated elapsed_ms and cleanup_verified in their receipt. + """ + + def __init__(self, control, campaign, search_id, snapshot, settings, adapter): + self.control, self.campaign, self.search_id = control, campaign, search_id + self.snapshot, self.settings, self.adapter = snapshot, settings, adapter + self.snapshot_id = digest(snapshot.document()) + self.settings_id = digest(asdict(settings)) + + def read(self): + document = self.control.read(self.campaign) + search = document["state"]["searches"][self.search_id] + if ( + search["snapshot"] != self.snapshot_id + or search["settings"] != self.settings_id + or search["adapter"] != self.adapter.identity + ): + raise ValueError("Frozen search roles, menu or adapter changed") + return document, search, decode(search["tree"]) + + def start(self, initial, *, seed, collect=True, forced=False): + if ( + initial.contract.search_id != self.search_id + or initial.contract.improver_snapshot != self.snapshot_id + ): + raise ValueError("Search root must bind the frozen search and improver") + tree.accept(initial) + if not forced and len(initial.candidates) != 1: + raise ValueError("Independent natural searches must start from one qualified root") + doc = self.control.read(self.campaign) + + def transition(state): + if self.search_id in state["searches"]: + raise ValueError("Search already exists; resume it") + state["searches"][self.search_id] = { + "tree": encode(initial), + "snapshot": self.snapshot_id, + "settings": self.settings_id, + "adapter": self.adapter.identity, + "seed": seed, + "collect": collect, + "forced": forced, + "trace": [], + "status": "active", + "pending": None, + "spent_ms": 0, + "started_at": time.time(), + "elapsed_ms": 0, + "draft_attempts": 0, + "active_action": None, + "initial_gain": tree.verified_gain(initial), + "selected": initial.root_id, + } + return state + + self.control.update(self.campaign, doc["revision"], transition) + + def _change(self, state, operation, result): + search = state["searches"][self.search_id] + pending = search["pending"] + if pending is None or pending["operation"] != operation: + raise ValueError("Late result does not match the active work identity") + current = decode(search["tree"]) + if pending["kind"] == "draft": + search["draft_attempts"] += 1 + if result.get("candidate") is not None: + raw = result["candidate"] + node = tree.Candidate( + **( + raw + | { + "producer": tree.Producer(**raw["producer"]), + "dependencies": tuple( + tree.Dependency(**d) for d in raw["dependencies"] + ), + } + ) + ) + try: + tree.accept(current) + root_valid = True + except ValueError: + root_valid = False + if root_valid and tree.candidate_valid(current, node.parent_id): + current = tree.add_candidate(current, node, expected_revision=current.revision) + else: + search.setdefault("rejected_drafts", []).append(operation) + else: + bound = next(a for a in current.attempts if a.attempt_id == operation) + from inferrsi.contracts.claim import Verdict + + receipt = tree.Receipt( + bound, + Verdict(result["verdict"]), + result["evidence_digest"], + result.get("objective"), + ) + current = tree.record_receipt(current, receipt, expected_revision=current.revision) + search["tree"] = encode(current) + if tree.complete_candidates(current) and tree.evidence(current, current.root_id): + try: + search["selected"] = tree.accept(current).candidate_id + except ValueError: + search.update(status="blocked", selected=None) + else: + search.update(status="blocked", selected=None) + search["spent_ms"] += result["elapsed_ms"] + search["pending"] = None + active = search["active_action"] + if active is not None: + active["cursor"] += 1 + if pending["kind"] == "draft": + active["last_child"] = ( + result["candidate"]["candidate_id"] + if result.get("candidate") + and any( + n.candidate_id == result["candidate"]["candidate_id"] + for n in current.candidates + ) + else None + ) + if result.get("choice") is not None: + search["trace"][-1]["draft_choices"].append(result["choice"]) + return state + + def invalidate(self, dependency): + """Persist invalidation and incumbent replacement before any cancellation. + + In-flight work remains charged to its original claim; late receipts can + still be archived by recover(), but can never revive invalid evidence. + """ + doc, _, _ = self.read() + + def transition(state): + record = state["searches"][self.search_id] + current = decode(record["tree"]) + current = tree.invalidate(current, dependency, expected_revision=current.revision) + record["tree"] = encode(current) + try: + record["selected"] = tree.accept(current).candidate_id + except ValueError: + record.update(status="blocked", selected=None) + record.setdefault("invalidations", []).append(dependency) + return state + + self.control.update(self.campaign, doc["revision"], transition) + + def recover(self): + _, search, _ = self.read() + pending = search["pending"] + if pending is None: + return + # Receipt-only recovery. Never call draft/verify from this path. + result = self.adapter.recover(pending["operation"], pending) + if result is None: + raise ValueError("Unknown execution: retain reservation and do not relaunch") + self.control.settle( + self.campaign, + pending["operation"], + result, + actual_ms=result.get("elapsed_ms"), + cleanup_verified=result.get("cleanup_verified"), + transition=lambda state: self._change(state, pending["operation"], result), + ) + + def _execute(self, kind, node, check=None): + doc, search, current = self.read() + operation = self.search_id + "-" + uuid.uuid4().hex + cap = self.settings.draft_ms if kind == "draft" else check.cost + binding = { + "search": self.search_id, + "candidate": asdict(node), + "check": check.check_id if check else None, + "snapshot": self.snapshot_id, + "adapter": self.adapter.identity, + } + + def pending(state): + search = state["searches"][self.search_id] + current = decode(search["tree"]) + if search["pending"] is not None: + raise ValueError("An operation is already pending") + if kind == "verify": + attempt = tree.Attempt( + operation, + operation, + operation, + operation, + self.search_id, + node.candidate_id, + node.artifact_digest, + check.check_id, + current.contract.context, + node.dependencies, + ) + current = tree.register_attempt( + current, attempt, expected_revision=current.revision + ) + search["tree"] = encode(current) + search["pending"] = {"operation": operation, "kind": kind, "binding": binding} + return state + + self.control.claim( + self.campaign, + doc["revision"], + operation, + category=kind, + cap_ms=cap, + binding=binding, + domains=self.adapter.domains(kind, node, check), + transition=pending, + ) + claim = self.control.launch(self.campaign, operation) + # Any exception retains the claim. The adapter's authenticated receipt is + # the only recovery route, including when writing the state below fails. + if kind == "draft": + rng = random.Random( + f"{search['seed']}:{len(current.candidates)}:{len(search['trace'])}" + ) + result = self.adapter.draft( + node, current, self.snapshot, rng, operation, claim["deadline"] + ) + else: + result = self.adapter.verify(node, check.check_id, operation, claim["deadline"]) + self.control.settle( + self.campaign, + operation, + result, + actual_ms=result.get("elapsed_ms"), + cleanup_verified=result.get("cleanup_verified"), + transition=lambda state: self._change(state, operation, result), + ) + _, updated, _ = self.read() + if updated["pending"] is not None: + raise ValueError("Execution or cleanup remains unknown") + return result + + def run(self, *, forced_action=None): + self.recover() + while True: + doc, search, current = self.read() + if search["status"] == "complete": + return search + if search["status"] == "blocked": + raise ValueError("Search root evidence is invalid; search remains blocked") + if doc["blocked"] or doc["final_claim"] is not None: + raise ValueError("Campaign blocked or final selection frozen") + if any( + op["status"] != "settled" and op["binding"].get("search") == self.search_id + for op in doc["operations"].values() + ): + raise ValueError("Unresolved claim: recover without relaunch") + if search["active_action"] is not None: + self._continue_action() + continue + remaining = min( + self.control.available(doc), self.settings.search_ms - search["spent_ms"] + ) + attempts_left = ( + current.contract.max_attempts - len(current.attempts) - search["draft_attempts"] + ) + if ( + len(search["trace"]) >= self.settings.max_steps + or attempts_left <= 0 + or remaining <= self.settings.preparation_ms + or time.time() >= doc["deadline"] + ): + return self.finish("limit") + op = self.search_id + "-prepare-" + uuid.uuid4().hex + self.control.claim( + self.campaign, + doc["revision"], + op, + category="preparation", + cap_ms=self.settings.preparation_ms, + binding={"search": self.search_id}, + ) + claim = self.control.launch(self.campaign, op) + started = time.monotonic() + jobs = self.adapter.jobs(current) + planning_state = replace( + current, + contract=replace( + current.contract, + max_attempts=current.contract.max_attempts - search["draft_attempts"], + ), + ) + actions = menu( + planning_state, + jobs, + remaining - self.settings.preparation_ms, + self.settings, + self.snapshot, + ) + actions = tuple( + a for a in actions if a.kind != "draft" or len(a.parents) * a.depth <= attempts_left + ) + vectors = tuple( + features(current, action, remaining, self.settings) for action in actions + ) + rng = random.Random(f"{search['seed']}:policy:{len(search['trace'])}") + choice = choose( + self.snapshot, + "policy", + [action.key for action in actions], + vectors, + rng=rng, + context=digest(encode(current)), + collect=search["collect"], + temperature=self.settings.temperature, + exploration=self.settings.exploration if search["collect"] else 0, + ) + selected = choice.selected + forced = forced_action is not None and not search["trace"] + if forced: + if not search["forced"]: + raise ValueError("Forced continuation must be isolated from actor training") + selected = next( + i for i, action in enumerate(actions) if action.key == forced_action + ) + action = actions[selected] + elapsed = max(1, math.ceil((time.monotonic() - started) * 1000)) + + def prepared(state): + record = state["searches"][self.search_id] + record["spent_ms"] += elapsed + record["trace"].append( + { + "action": asdict(action), + "choice": asdict(choice), + "forced": forced, + "features": vectors[selected], + "draft_choices": [], + "gain_before": tree.verified_gain(current), + "spent_before": record["spent_ms"], + "at": time.time(), + "paid_state": encode(current), + "menu": [asdict(a) for a in actions], + "vectors": vectors, + "jobs": [asdict(job) for job in jobs], + "draft_attempts": record["draft_attempts"], + } + ) + if action.kind != "accept": + record["active_action"] = { + "action": asdict(action), + "cursor": 0, + "last_child": None, + "jobs": [asdict(job) for job in jobs], + } + return state + + self.control.settle( + self.campaign, + op, + {"menu": [asdict(a) for a in actions], "elapsed_ms": elapsed}, + actual_ms=elapsed, + cleanup_verified=True, + transition=prepared, + ) + if time.time() >= claim["deadline"]: + raise ValueError("Preparation deadline exceeded") + if action.kind == "accept": + return self.finish("accept") + + def _continue_action(self): + from .verification_plans import Job + + while True: + doc, search, current = self.read() + active = search["active_action"] + action = Action(**active["action"]) + cursor = active["cursor"] + limit = ( + len(action.parents) * action.depth if action.kind == "draft" else len(action.jobs) + ) + if cursor >= limit: + + def done(state): + state["searches"][self.search_id]["active_action"] = None + return state + + self.control.update(self.campaign, doc["revision"], done) + return + next_cap = ( + self.settings.draft_ms + if action.kind == "draft" + else next( + job["cost"] for job in active["jobs"] if job["job_id"] == action.jobs[cursor] + ) + ) + if ( + time.time() >= doc["deadline"] + or next_cap > self.control.available(doc) + or next_cap > self.settings.search_ms - search["spent_ms"] + ): + + def exhausted(state): + record = state["searches"][self.search_id] + record["active_action"] = None + record["block_stop_reason"] = "budget_or_deadline" + return state + + self.control.update(self.campaign, doc["revision"], exhausted) + return + if action.kind == "draft": + branch, level = divmod(cursor, action.depth) + if level and active["last_child"] is None: + + def skip(state): + state["searches"][self.search_id]["active_action"]["cursor"] = ( + branch + 1 + ) * action.depth + return state + + self.control.update(self.campaign, doc["revision"], skip) + continue + parent = active["last_child"] if level else action.parents[branch] + self._execute("draft", tree.candidate(current, parent)) + else: + by_id = {raw["job_id"]: Job(**raw) for raw in active["jobs"]} + job = by_id[action.jobs[cursor]] + if job.candidate_id is None: + raise ValueError("Adapter must package shared setup into its real check") + self._execute("verify", tree.candidate(current, job.candidate_id), job) + + def finish(self, reason): + doc, search, current = self.read() + if search["pending"] is not None: + raise ValueError("Cannot accept unsettled work") + if search["active_action"] is not None: + raise ValueError("Cannot accept an unfinished bounded work block") + if any( + op["status"] != "settled" and op["binding"].get("search") == self.search_id + for op in doc["operations"].values() + ): + raise ValueError("Cannot accept an unresolved claim") + + def transition(state): + record = state["searches"][self.search_id] + record.update( + status="complete", + stop_reason=reason, + selected=tree.accept(current).candidate_id, + gain=tree.verified_gain(current), + elapsed_ms=max(1, math.ceil((time.time() - record["started_at"]) * 1000)), + ) + return state + + self.control.update(self.campaign, doc["revision"], transition) + return self.read()[1] diff --git a/src/inferrsi/harness/work_control.py b/src/inferrsi/harness/work_control.py new file mode 100644 index 00000000..3a0993c4 --- /dev/null +++ b/src/inferrsi/harness/work_control.py @@ -0,0 +1,278 @@ +"""Campaign accounting and CAS in the shared RunStore database. + +This is a controller for trusted host operations, not an executor or sandbox. +All cooperating controllers must share the same RunStore for resource domains +and holdout scopes to be exclusive. Unknown work retains its entire reservation. +""" + +import json +import math +import time +from contextlib import contextmanager + +from .artifact_primitives import digest +from .run_store import _transaction + + +def _json(value): + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + + +def _positive(value): + if type(value) is not int or value <= 0: + raise ValueError("Work limits must be positive integer milliseconds") + + +class WorkControl: + def __init__(self, store): + self.store = store + if store._read_only: + return + with store._connection() as db, _transaction(db): + db.execute( + "CREATE TABLE IF NOT EXISTS work_campaigns " + "(id TEXT PRIMARY KEY, revision INTEGER NOT NULL, body TEXT NOT NULL)" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS work_domains " + "(id TEXT PRIMARY KEY, campaign TEXT NOT NULL, operation TEXT NOT NULL)" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS work_holdouts " + "(scope TEXT PRIMARY KEY, campaign TEXT NOT NULL, binding TEXT NOT NULL)" + ) + + @contextmanager + def _transaction(self, campaign, revision=None): + with self.store._connection() as db, _transaction(db): + row = db.execute( + "SELECT revision, body FROM work_campaigns WHERE id=?", (campaign,) + ).fetchone() + if row is None: + raise ValueError("Unknown campaign") + if revision is not None and row[0] != revision: + raise ValueError("Stale campaign revision") + body = json.loads(row[1]) + yield db, body + db.execute( + "UPDATE work_campaigns SET revision=?, body=? WHERE id=?", + (row[0] + 1, _json(body), campaign), + ) + + def create(self, campaign, contract, state, *, cap_ms, final_reserve_ms, deadline): + _positive(cap_ms) + _positive(final_reserve_ms) + if ( + not campaign + or final_reserve_ms >= cap_ms + or not math.isfinite(deadline) + or deadline <= time.time() + ): + raise ValueError("Invalid campaign budget or deadline") + body = { + "contract": contract, + "contract_digest": digest(contract), + "state": state, + "cap_ms": cap_ms, + "final_reserve_ms": final_reserve_ms, + "deadline": deadline, + "spent_ms": 0, + "operations": {}, + "events": [], + "final_claim": None, + "final_result": None, + "blocked": False, + } + with self.store._connection() as db, _transaction(db): + db.execute("INSERT INTO work_campaigns VALUES (?, 0, ?)", (campaign, _json(body))) + return self.read(campaign) + + def read(self, campaign): + with self.store._connection() as db: + row = db.execute( + "SELECT revision, body FROM work_campaigns WHERE id=?", (campaign,) + ).fetchone() + if row is None: + raise ValueError("Unknown campaign") + body = json.loads(row[1]) + if digest(body["contract"]) != body["contract_digest"]: + raise ValueError("Frozen campaign contract changed") + return {"revision": row[0], **body} + + @staticmethod + def available(body, *, final=False): + held = sum(op["cap_ms"] for op in body["operations"].values() if op["status"] != "settled") + return body["cap_ms"] - body["spent_ms"] - held - (0 if final else body["final_reserve_ms"]) + + def update(self, campaign, revision, transition): + """Persist a pure state transition and incumbent recomputation in one CAS.""" + with self._transaction(campaign, revision) as (_, body): + if body["blocked"] or body["final_claim"] is not None: + raise ValueError("Campaign transitions are blocked or final selection is frozen") + body["state"] = transition(body["state"]) + return self.read(campaign) + + def claim( + self, + campaign, + revision, + operation, + *, + category, + cap_ms, + domains=(), + binding, + final=False, + transition=None, + ): + _positive(cap_ms) + domains = tuple(sorted(set(domains))) + if not operation or not category or any(not item for item in domains): + raise ValueError("Missing operation or resource identity") + if final != (category == "final"): + raise ValueError("Final reservations require the final work category") + with self._transaction(campaign, revision) as (db, body): + if operation in body["operations"]: + raise ValueError("Operation already claimed; recover without relaunch") + if body["blocked"] or (body["final_claim"] is not None and not final): + raise ValueError("Campaign is blocked or final selection is frozen") + if final and body["final_claim"] is None: + raise ValueError("Final work requires a durable one-time claim") + if time.time() >= body["deadline"] or cap_ms > self.available(body, final=final): + raise ValueError("Work exceeds remaining budget or deadline") + for domain in domains: + if db.execute("SELECT 1 FROM work_domains WHERE id=?", (domain,)).fetchone(): + raise ValueError("Measurement domain is already leased") + db.execute( + "INSERT INTO work_domains VALUES (?, ?, ?)", (domain, campaign, operation) + ) + body["operations"][operation] = { + "category": category, + "binding": binding, + "cap_ms": cap_ms, + "domains": domains, + "status": "claimed", + "receipt": None, + "deadline": min(body["deadline"], time.time() + cap_ms / 1000), + "execution_key": operation, + } + body["events"].append({"kind": "claimed", "operation": operation, "at": time.time()}) + if transition is not None: + body["state"] = transition(body["state"]) + return self.read(campaign) + + def launch(self, campaign, operation): + """Persist intent before external launch. A crash here must never relaunch.""" + expired = False + with self._transaction(campaign) as (db, body): + op = body["operations"][operation] + if op["status"] != "claimed": + raise ValueError("Execution intent already recorded") + expired = time.time() >= op["deadline"] + if expired: + op.update(status="settled", actual_ms=0, receipt={"not_started": "deadline"}) + db.execute( + "DELETE FROM work_domains WHERE campaign=? AND operation=?", + (campaign, operation), + ) + else: + op.update(status="launched", started_at=time.time()) + body["events"].append( + { + "kind": "not_started" if expired else "launch_intent", + "operation": operation, + "at": time.time(), + } + ) + if expired: + raise ValueError("Deadline expired before launch") + return self.read(campaign)["operations"][operation] + + def settle(self, campaign, operation, receipt, *, actual_ms, cleanup_verified, transition=None): + """Only authenticated, owner-scoped receipts may be passed by the adapter. + + Receipt replay is idempotent. Evidence and accounting commit together. + Unknown usage/cleanup does not release the domain or budget reservation. + """ + if actual_ms is not None and (type(actual_ms) is not int or actual_ms < 0): + raise ValueError("Usage must be nonnegative integer milliseconds or unknown") + with self._transaction(campaign) as (db, body): + op = body["operations"][operation] + settlement = { + "receipt": receipt, + "actual_ms": actual_ms, + "cleanup_verified": cleanup_verified, + } + if op["status"] == "settled": + if _json(op.get("settlement")) != _json(settlement): + raise ValueError("Conflicting duplicate settlement") + return + if op["status"] not in ("launched", "unknown"): + raise ValueError("Cannot settle an unlaunched operation") + op["settlement"] = settlement + if actual_ms is None or cleanup_verified is not True: + op["status"] = "unknown" + body["events"].append( + {"kind": "unknown", "operation": operation, "at": time.time()} + ) + else: + op.update(status="settled", actual_ms=actual_ms, receipt=receipt) + body["spent_ms"] += actual_ms + executable_ms = min(op["cap_ms"], (op["deadline"] - op["started_at"]) * 1000) + if actual_ms > executable_ms or body["spent_ms"] > body["cap_ms"]: + body["blocked"] = True + if transition is not None and not body["blocked"]: + body["state"] = transition(body["state"]) + db.execute( + "DELETE FROM work_domains WHERE campaign=? AND operation=?", + (campaign, operation), + ) + body["events"].extend( + [ + {"kind": "cleanup_confirmed", "operation": operation, "at": time.time()}, + {"kind": "released", "operation": operation, "at": time.time()}, + ] + ) + return self.read(campaign) + + def claim_final(self, campaign, revision, scope, binding): + """Consume a shared holdout scope before any access, including interrupted access.""" + if not isinstance(scope, str) or not scope.strip(): + raise ValueError("Holdout scope must have a nonempty identity") + with self._transaction(campaign, revision) as (db, body): + if ( + body["final_claim"] is not None + or body["blocked"] + or any(op["status"] != "settled" for op in body["operations"].values()) + ): + raise ValueError("Final is already claimed or work remains unresolved") + if time.time() >= body["deadline"]: + raise ValueError("Final deadline exhausted") + if db.execute("SELECT 1 FROM work_holdouts WHERE scope=?", (scope,)).fetchone(): + raise ValueError("Holdout scope already consumed in this RunStore") + db.execute( + "INSERT INTO work_holdouts VALUES (?, ?, ?)", (scope, campaign, _json(binding)) + ) + body["final_claim"] = {"scope": scope, "binding": binding} + return self.read(campaign) + + def finish_final(self, campaign, binding, result, *, transition=None): + with self._transaction(campaign) as (_, body): + if body["final_claim"] is None or _json(body["final_claim"]["binding"]) != _json( + binding + ): + raise ValueError("Final result does not match the frozen selection") + if body["final_result"] is not None: + if _json(body["final_result"]) != _json(result): + raise ValueError("Final result is immutable; export the existing result") + return + if transition is not None and ( + body["blocked"] + or any(op["status"] != "settled" for op in body["operations"].values()) + or result.get("verdict") != "PASS" + ): + raise ValueError("Unresolved or over-budget work cannot promote a candidate") + body["final_result"] = result + if transition is not None: + body["state"] = transition(body["state"]) + return self.read(campaign) diff --git a/src/inferrsi/model/search_roles.py b/src/inferrsi/model/search_roles.py new file mode 100644 index 00000000..3e5df28c --- /dev/null +++ b/src/inferrsi/model/search_roles.py @@ -0,0 +1,215 @@ +"""Trainable finite-edit Drafter, independent Policy and linear continuation Scorer. + +The reference CPU model samples tokens from a frozen vocabulary of permitted edit +operators. It is not a language-model fine-tuner. Hosts may supply another role +implementation, but must preserve observed contexts, masks and behavior laws. +""" + +from dataclasses import asdict, dataclass, replace +from math import exp, fsum, isfinite, log + + +def finite(value): + if type(value) not in (float, int) or not isfinite(value): + raise ValueError("Expected a finite numeric observation") + return value + + +def dot(left, right): + if len(left) != len(right) or not left: + raise ValueError("Role feature dimension mismatch") + return finite(fsum(finite(a) * finite(b) for a, b in zip(left, right))) + + +def distribution(weights, features, temperature=1.0, exploration=0.0): + if finite(temperature) <= 0 or not 0 <= finite(exploration) < 1 or not features: + raise ValueError("Invalid sampling law") + logits = [dot(weights, row) / temperature for row in features] + scale = max(logits) + raw = [exp(value - scale) for value in logits] + total = fsum(raw) + soft = tuple(value / total for value in raw) + probabilities = tuple((1 - exploration) * p + exploration / len(soft) for p in soft) + if any(p <= 0 for p in probabilities): + raise ValueError("Sampling law underflows; rescale model or increase exploration") + return probabilities, soft + + +@dataclass(frozen=True) +class Snapshot: + drafter: tuple[float, ...] + policy: tuple[float, ...] + scorer: tuple[float, ...] + origin: str + + def __post_init__(self): + for name in ("drafter", "policy", "scorer"): + values = tuple(getattr(self, name)) + if not values: + raise ValueError("Empty role parameters") + for value in values: + finite(value) + object.__setattr__(self, name, values) + if not self.origin: + raise ValueError("Checkpoint provenance is required") + + def document(self): + return { + **asdict(self), + "drafter": list(self.drafter), + "policy": list(self.policy), + "scorer": list(self.scorer), + } + + +@dataclass(frozen=True) +class Choice: + role: str + options: tuple[str, ...] + features: tuple[tuple[float, ...], ...] + selected: int + probabilities: tuple[float, ...] + temperature: float + exploration: float + context: str + sampled: bool = True + + def __post_init__(self): + if ( + self.role not in ("drafter", "policy") + or not self.context + or len(set(self.options)) != len(self.options) + or not 0 <= self.selected < len(self.options) + or len(self.options) != len(self.features) + or len(self.options) != len(self.probabilities) + or abs(fsum(self.probabilities) - 1) > 1e-10 + or any(not 0 < finite(p) <= 1 for p in self.probabilities) + ): + raise ValueError("Invalid recorded role choice") + + +def choose( + snapshot, role, options, features, *, rng, context, collect, temperature=1.0, exploration=0.0 +): + probabilities, _ = distribution(getattr(snapshot, role), features, temperature, exploration) + if collect or role == "drafter": + selected = rng.choices(range(len(options)), weights=probabilities, k=1)[0] + else: + selected = max(range(len(options)), key=lambda index: probabilities[index]) + return Choice( + role, + tuple(options), + tuple(tuple(row) for row in features), + selected, + probabilities, + temperature, + exploration, + context, + sampled=collect or role == "drafter", + ) + + +def actor_gradient(weights, reference, choice, advantage, clip, kl_weight): + """Gradient ascent for the clipped surrogate minus KL to a frozen reference.""" + probabilities, soft = distribution( + weights, choice.features, choice.temperature, choice.exploration + ) + refs, _ = distribution(reference, choice.features, choice.temperature, choice.exploration) + ratio = finite(probabilities[choice.selected] / choice.probabilities[choice.selected]) + active = not ((advantage > 0 and ratio > 1 + clip) or (advantage < 0 and ratio < 1 - clip)) + means = [fsum(p * row[d] for p, row in zip(soft, choice.features)) for d in range(len(weights))] + jacobian = [ + tuple( + (1 - choice.exploration) * p * (row[d] - means[d]) / choice.temperature + for d in range(len(weights)) + ) + for p, row in zip(soft, choice.features) + ] + return tuple( + finite( + ( + advantage / choice.probabilities[choice.selected] * jacobian[choice.selected][d] + if active + else 0 + ) + - kl_weight + * fsum( + jacobian[i][d] * (log(probabilities[i] / refs[i]) + 1) + for i in range(len(probabilities)) + ) + ) + for d in range(len(weights)) + ) + + +def update( + snapshot, + reference, + actor_examples, + value_examples, + pair_examples=(), + *, + learning_rate, + clip=0.2, + kl_weight=0.01, + pair_weight=1.0, + epochs=1, + groups=1, +): + """Update actual parameters; forced trajectories belong only in value/pair examples. + + Actor examples are (Choice, whole-search advantage), value examples are + (features, realized suffix return), pairs are (left features, right features, + paid observed return difference). Missing targets are rejected, never imputed. + """ + if ( + finite(learning_rate) <= 0 + or not 0 < finite(clip) < 1 + or finite(kl_weight) < 0 + or finite(pair_weight) < 0 + or type(epochs) is not int + or epochs < 1 + or type(groups) is not int + or groups < 1 + ): + raise ValueError("Invalid frozen training settings") + actors, values, pairs = tuple(actor_examples), tuple(value_examples), tuple(pair_examples) + for choice, advantage in actors: + finite(advantage) + if not choice.sampled: + raise ValueError("Deterministic actions cannot enter the actor objective") + replay, _ = distribution( + getattr(snapshot, choice.role), choice.features, choice.temperature, choice.exploration + ) + if any(abs(a - b) > 1e-10 for a, b in zip(replay, choice.probabilities)): + raise ValueError("Behavior probabilities do not match the frozen checkpoint") + current = snapshot + for _ in range(epochs): + changed = {} + for role in ("drafter", "policy"): + weights = getattr(current, role) + gradients = [ + actor_gradient( + weights, getattr(reference, role), choice, advantage, clip, kl_weight + ) + for choice, advantage in actors + if choice.role == role + ] + changed[role] = tuple( + finite(w + learning_rate * fsum(g[d] for g in gradients) / groups) + for d, w in enumerate(weights) + ) + weights = current.scorer + gradient = [0.0] * len(weights) + for features, target in values: + error = dot(weights, features) - finite(target) + for d in range(len(weights)): + gradient[d] += 2 * error * features[d] / len(values) + for left, right, target in pairs: + delta = tuple(a - b for a, b in zip(left, right)) + error = dot(weights, delta) - finite(target) + for d in range(len(weights)): + gradient[d] += pair_weight * 2 * error * delta[d] / len(pairs) + changed["scorer"] = tuple(finite(w - learning_rate * g) for w, g in zip(weights, gradient)) + current = replace(current, **changed) + return current diff --git a/tests/test_search_benchmarks.py b/tests/test_search_benchmarks.py new file mode 100644 index 00000000..5cc1ba6b --- /dev/null +++ b/tests/test_search_benchmarks.py @@ -0,0 +1,115 @@ +import json +import time +from types import SimpleNamespace + +import pytest + +from inferrsi.apps.cli.arguments import parser +from inferrsi.apps.search_benchmarks import EngineEvaluator, SkillEvaluator +from inferrsi.apps.speculative_search import command +from inferrsi.benchmark.improvement import SelectionPolicy + + +def skill_report(base_tokens=100, tokens=80, *, verdict="PASS", outcome="unchanged"): + def trial(count, result): + return { + "verdict": result, + "elapsed_seconds": 1, + "observations": { + "usage": None + if count is None + else {"input_tokens": count, "output_tokens": 0, "cached_input_tokens": 0}, + "tool_calls": 0, + }, + } + + return { + "status": {"execution_state": "completed"}, + "pairs": [ + { + "baseline": trial(base_tokens, "PASS"), + "candidate": trial(tokens, verdict), + "outcome": outcome, + } + ], + } + + +@pytest.mark.parametrize( + "report,split,verdict", + [ + (skill_report(), "development", "PASS"), + (skill_report(tokens=1, verdict="FAIL", outcome="regression"), "development", "FAIL"), + (skill_report(base_tokens=None), "development", "INCONCLUSIVE"), + (skill_report(tokens=None), "development", "INCONCLUSIVE"), + (skill_report(tokens=101), "holdout", "FAIL"), + (skill_report(tokens=100), "holdout", "PASS"), + ], +) +def test_skill_quality_unknown_usage_and_independent_holdout_gates(report, split, verdict): + comparator = SimpleNamespace(identity="explicit-comparator", run=lambda *a, **k: (report, True)) + evaluator = SkillEvaluator( + comparator, split=split, policy=SelectionPolicy(1, 0), token_cap=1000, task_count=1 + ) + result = evaluator.evaluate(None, "evaluation", None, time.time() + 1) + assert result["verdict"] == verdict + assert (result["objective"] is not None) == (verdict == "PASS") + + +def test_engine_uses_frozen_complete_task_protocol_and_rejects_mutation(tmp_path): + evaluator = EngineEvaluator( + "qwen21-denoise-npu", + tmp_path / "frozen", + arguments=["--pairs", "1", "--samples", "1", "--warmups", "0"], + ) + assert evaluator.plan["evaluation_settings"] == {"pairs": 1, "samples": 1, "warmups": 0} + (evaluator.frozen / "reference.py").write_text("changed reference") + with pytest.raises(ValueError, match="resources changed"): + evaluator.evaluate(tmp_path, "evaluation", tmp_path / "output", time.time() + 1) + assert not (tmp_path / "output").exists() + + +def test_cli_requires_explicit_shared_store_and_read_commands_do_not_create_one(tmp_path): + args = parser().parse_args(["search", "status", "--session", str(tmp_path / "absent")]) + with pytest.raises(FileNotFoundError): + command(args) + assert not (tmp_path / "absent").exists() + with pytest.raises(SystemExit): + parser().parse_args(["search", "engine", "--session", str(tmp_path / "run")]) + + +def test_skill_comparator_preparation_does_not_access_holdout(tmp_path): + from tests.fixtures.harness_improvement_demo import configuration + from tools.speculative_skill import Comparator + + config, repo = configuration(tmp_path) + comparison = {k: config[k] for k in ("skill", "skill_root", "model", "limits")} + comparison.update( + mode="update", + client={"explicit": "preparation-only-test"}, + snapshots={"baseline": {"git_revision": config["baseline"]}}, + ) + def descriptor(split): + return { + "path": str(tmp_path / (split + ".json")), + "sha256": split, + "scope": split, + "task_count": 3, + } + bound = Comparator( + repo, + { + "comparison": comparison, + "allowed_paths": config["allowed_paths"], + "splits": {s: descriptor(s) for s in ("development", "holdout")}, + }, + tmp_path / "baseline-snapshot", + ) + exported = bound.export_root(tmp_path / "candidate") + assert (exported / "selected/SKILL.md").is_file() + assert not (tmp_path / "holdout.json").exists() + assert not (tmp_path / "development.json").exists() + # The prepared adapter still fails before provider access if its split is missing. + with pytest.raises(FileNotFoundError): + bound.run(exported, tmp_path / "run", time.time() + 1, split="development") + json.dumps(bound.config) diff --git a/tests/test_search_roles.py b/tests/test_search_roles.py new file mode 100644 index 00000000..3378e7be --- /dev/null +++ b/tests/test_search_roles.py @@ -0,0 +1,71 @@ +import random + +import pytest + +from inferrsi.model.search_roles import Snapshot, actor_gradient, choose, distribution, update + + +def test_actor_gradient_matches_finite_difference_with_exploration_and_kl(): + model = Snapshot((0.2, -0.1), (0.3, -0.2), (0.0, 0.0), "initial") + choice = choose( + model, + "policy", + ("a", "b"), + ((1.0, 0.0), (0.0, 1.0)), + rng=random.Random(2), + context="observed", + collect=True, + exploration=0.1, + ) + reference = (0.0, 0.0) + gradient = actor_gradient(model.policy, reference, choice, 2.0, 0.2, 0.1) + from math import log + + def objective(weights): + p, _ = distribution(weights, choice.features, exploration=0.1) + q, _ = distribution(reference, choice.features, exploration=0.1) + return 2 * p[choice.selected] / choice.probabilities[choice.selected] - 0.1 * sum( + a * log(a / b) for a, b in zip(p, q) + ) + + for i in range(2): + plus, minus = list(model.policy), list(model.policy) + plus[i] += 1e-6 + minus[i] -= 1e-6 + assert gradient[i] == pytest.approx((objective(plus) - objective(minus)) / 2e-6) + + +def test_actual_updates_keep_policy_scorer_independent_and_reject_false_behavior(): + model = Snapshot((0.0, 0.0), (0.0, 0.0), (0.0, 0.0), "initial") + choice = choose( + model, + "drafter", + ("a", "b"), + ((1.0, 0.0), (0.0, 1.0)), + rng=random.Random(1), + context="observed", + collect=True, + ) + trained = update(model, model, [(choice, 2.0)], [((1.0, 0.0), 4.0)], learning_rate=0.1) + assert trained.drafter != model.drafter + assert trained.scorer != model.scorer + assert trained.policy == model.policy + with pytest.raises(ValueError, match="Behavior"): + update(trained, model, [(choice, 2.0)], [], learning_rate=0.1) + with pytest.raises(ValueError, match="finite"): + update(model, model, [], [((1.0, 0.0), None)], learning_rate=0.1) + + +def test_clipped_actor_does_not_reinforce_outside_positive_ratio_bound(): + model = Snapshot((0.0, 0.0), (0.0, 0.0), (0.0, 0.0), "initial") + choice = choose( + model, + "policy", + ("a", "b"), + ((1.0, 0.0), (0.0, 1.0)), + rng=random.Random(1), + context="observed", + collect=True, + ) + weights = (10.0, -10.0) if choice.selected == 0 else (-10.0, 10.0) + assert actor_gradient(weights, model.policy, choice, 1.0, 0.2, 0.0) == (0.0, 0.0) diff --git a/tests/test_speculative_search.py b/tests/test_speculative_search.py index 997e95f5..ad686fbf 100644 --- a/tests/test_speculative_search.py +++ b/tests/test_speculative_search.py @@ -63,6 +63,14 @@ def test_objective_prediction_partial_results_and_failed_quality_cannot_win(): assert tree.verified_gain(state) == 0 +@pytest.mark.parametrize("direction,sign", [("maximize", 1), ("minimize", -1)]) +def test_finite_objectives_cannot_publish_an_infinite_gain(direction, sign): + state = start(configuration=contract(direction=direction), score=-sign * 1e308) + state = complete(add(state, "candidate"), "candidate", sign * 1e308) + with pytest.raises(ValueError, match="finite objective"): + tree.verified_gain(state) + + @pytest.mark.parametrize("direction", ["maximize", "minimize"]) @pytest.mark.parametrize("order", [("z", "a"), ("a", "z")]) def test_selection_is_independent_of_receipt_order_and_root_wins_ties(direction, order): diff --git a/tests/test_vats_runtime.py b/tests/test_vats_runtime.py new file mode 100644 index 00000000..bd3a793f --- /dev/null +++ b/tests/test_vats_runtime.py @@ -0,0 +1,310 @@ +import json +import sys +import time +from dataclasses import asdict, replace + +import pytest + +from inferrsi.harness import search_tree as tree +from inferrsi.harness.artifact_primitives import digest +from inferrsi.harness.gvo import examples, train_group +from inferrsi.harness.run_store import RunStore +from inferrsi.harness.search_adapter import ArtifactAdapter, CommandEvaluator, Edit +from inferrsi.harness.search_campaign import collect_group, finalize, qualify_root +from inferrsi.harness.search_records import decode +from inferrsi.harness.vats import Search, Settings +from inferrsi.harness.work_control import WorkControl +from inferrsi.model.search_roles import Snapshot +from tests.fixtures.speculative_search import contract + + +@pytest.fixture +def runtime(tmp_path): + grader = tmp_path / "grader.py" + grader.write_text("""import json, pathlib, sys +value = int((pathlib.Path(sys.argv[1]) / "candidate.txt").read_text()) +check = sys.argv[2] +out = {"verdict": "PASS" if value > 0 else "FAIL"} +if value > 0 and check == "objective": + out["objective"] = value +pathlib.Path(sys.argv[3]).write_text(json.dumps(out)) +""") + source = tmp_path / "baseline" + source.mkdir() + (source / "candidate.txt").write_text("1") + evaluator = CommandEvaluator( + { + check: (sys.executable, str(grader), "{candidate}", check, "{result}") + for check in ("quality", "objective") + }, + [grader], + ) + adapter = ArtifactAdapter( + tmp_path / "artifacts", + ( + Edit("improve", "candidate.txt", "1", "3"), + Edit("break", "candidate.txt", "1", "-1"), + Edit("repair", "candidate.txt", "-1", "4"), + ), + {"quality": 3000, "objective": 3000}, + evaluator, + domain="cpu-tests", + ) + snapshot = Snapshot( + (1.0, 0.0, 0.0), + (-1.0, 3.0, 6.0, 0.0, 0.0, 0.0, 0.0, 0.0), + (0.0,) * 8, + "test-initial-checkpoint", + ) + settings = Settings(5, 1, 2, 2, 40, 50, 3000, 3000, 40000) + control = WorkControl(RunStore(tmp_path / "store", "search-test")) + control.create( + "campaign", + {"adapter": adapter.identity, "settings": asdict(settings)}, + {"searches": {}, "checkpoint": snapshot.document()}, + cap_ms=120000, + final_reserve_ms=12000, + deadline=time.time() + 120, + ) + root = adapter.import_root(source, snapshot=digest(snapshot.document()), context={}) + initial = qualify_root( + control, "campaign", adapter, root, contract(improver_snapshot=digest(snapshot.document())) + ) + return control, adapter, snapshot, settings, initial + + +def test_real_process_search_group_training_checkpoint_and_fresh_search(runtime): + control, adapter, snapshot, settings, root = runtime + records = collect_group( + control, "campaign", adapter, root, snapshot, settings, group_id="first", size=2, seed=12 + ) + assert all(record["status"] == "complete" for record in records) + assert all(tree.accept(decode(record["tree"])) for record in records) + assert any(len(record["tree"]["candidates"]) > 1 for record in records) + actors, values, group = examples(records) + assert actors and values and len(group["advantages"]) == 2 + trained = train_group( + control, "campaign", records, snapshot, snapshot, cap_ms=5000, learning_rate=0.01 + ) + assert trained.document() == control.read("campaign")["state"]["checkpoint"] + assert trained.scorer != snapshot.scorer + next_records = collect_group( + control, "campaign", adapter, root, trained, settings, group_id="next", size=2, seed=42 + ) + assert all(record["snapshot"] == digest(trained.document()) for record in next_records) + assert control.read("campaign")["spent_ms"] > 0 + + +def test_final_access_is_frozen_and_export_retry_does_not_execute(runtime): + control, adapter, snapshot, settings, root = runtime + records = collect_group( + control, "campaign", adapter, root, snapshot, settings, group_id="first", size=2, seed=4 + ) + selected = tree.accept(decode(records[0]["tree"])) + result = finalize( + control, + "campaign", + adapter, + selected, + holdout_scope="test-final", + accept_result=lambda results: all(r["verdict"] == "PASS" for r in results), + ) + before = len(list((adapter.root / "executions").iterdir())) + assert ( + finalize( + control, + "campaign", + adapter, + selected, + holdout_scope="test-final", + accept_result=lambda results: False, + ) + == result + ) + assert len(list((adapter.root / "executions").iterdir())) == before + assert control.read("campaign")["state"]["stable"]["candidate_id"] == selected.candidate_id + + +def test_resume_after_receipt_before_state_commit_never_executes_twice(runtime, monkeypatch): + control, adapter, snapshot, settings, root = runtime + search = Search(control, "campaign", root.contract.search_id, snapshot, settings, adapter) + search.start(root, seed=2, collect=False) + original = control.settle + failed = False + + def crash(campaign, operation, receipt, **kwargs): + nonlocal failed + if receipt.get("candidate") and not failed: + failed = True + raise RuntimeError("crash before commit") + return original(campaign, operation, receipt, **kwargs) + + monkeypatch.setattr(control, "settle", crash) + with pytest.raises(RuntimeError, match="crash"): + search.run() + assert search.read()[1]["pending"] is not None + monkeypatch.setattr(control, "settle", original) + before = len(list((adapter.root / "candidates").iterdir())) + search.recover() + assert search.read()[1]["pending"] is None + assert len(list((adapter.root / "candidates").iterdir())) == before + + +def test_gvo_rejects_shared_tree_leaves_forced_and_changed_start_conditions(runtime): + control, adapter, snapshot, settings, root = runtime + records = collect_group( + control, "campaign", adapter, root, snapshot, settings, group_id="group", size=2, seed=2 + ) + with pytest.raises(ValueError, match="distinct"): + examples((records[0], records[0])) + with pytest.raises(ValueError, match="Forced"): + examples((records[0] | {"forced": True}, records[1])) + with pytest.raises(ValueError, match="identical"): + examples((records[0] | {"settings": "other"}, records[1])) + + +def test_snapshot_change_on_resume_is_rejected(runtime): + control, adapter, snapshot, settings, root = runtime + Search(control, "campaign", "search", snapshot, settings, adapter).start(root, seed=0) + changed = replace(snapshot, policy=(2.0,) * 8) + with pytest.raises(ValueError, match="Frozen"): + Search(control, "campaign", "search", changed, settings, adapter).run() + + +def test_persisted_search_has_only_real_check_results(runtime): + control, adapter, snapshot, settings, root = runtime + records = collect_group( + control, "campaign", adapter, root, snapshot, settings, group_id="group", size=2, seed=5 + ) + serialized = json.dumps(control.read("campaign"), allow_nan=False) + assert serialized + for record in records: + state = decode(record["tree"]) + for node in tree.complete_candidates(state): + assert all( + r.verdict.value == "PASS" for r in tree.evidence(state, node.candidate_id).values() + ) + + +def test_paid_continuations_are_metered_matched_and_excluded_from_actors(runtime): + from inferrsi.harness.search_continuations import continue_from, paired_targets + from inferrsi.harness.vats import Action + + control, adapter, snapshot, settings, root = runtime + records = collect_group( + control, "campaign", adapter, root, snapshot, settings, group_id="natural", size=2, seed=12 + ) + source = Search(control, "campaign", "natural-0", snapshot, settings, adapter) + menu = [Action(**a) for a in records[0]["trace"][0]["menu"]] + accept = next(a for a in menu if a.kind == "accept") + draft = next(a for a in menu if a.kind == "draft") + before = control.read("campaign")["spent_ms"] + suffixes = [ + continue_from(source, 0, a.key, f"suffix-{i}", copy_cap_ms=3000, seed=91) + for i, a in enumerate((accept, draft)) + ] + assert control.read("campaign")["spent_ms"] > before + assert all(s["forced"] and s["status"] == "complete" for s in suffixes) + pair = paired_targets(suffixes, expense_weight=0.1, elapsed_weight=0) + assert len(pair) == 3 + assert any( + op["category"] == "continuation_copy" + for op in control.read("campaign")["operations"].values() + ) + with pytest.raises(ValueError, match="Forced"): + examples(suffixes) + trained = train_group( + control, + "campaign", + records, + snapshot, + snapshot, + cap_ms=5000, + learning_rate=0.01, + scorer_continuations=suffixes, + scorer_pairs=(suffixes,), + ) + assert trained.scorer != snapshot.scorer + + +def test_root_invalidation_is_persisted_and_cannot_return_an_invalid_root(runtime): + control, adapter, snapshot, settings, root = runtime + dependency = tree.Dependency("environment", 0) + nodes = tuple(replace(n, dependencies=(dependency,)) for n in root.candidates) + attempts = tuple(replace(a, dependencies=(dependency,)) for a in root.attempts) + by_id = {a.attempt_id: a for a in attempts} + receipts = tuple(replace(r, attempt=by_id[r.attempt.attempt_id]) for r in root.receipts) + initial = replace( + root, candidates=nodes, attempts=attempts, receipts=receipts, dependencies=(dependency,) + ) + search = Search(control, "campaign", "search", snapshot, settings, adapter) + search.start(initial, seed=1) + search.invalidate("environment") + assert search.read()[1]["selected"] is None + assert search.read()[1]["status"] == "blocked" + with pytest.raises(ValueError, match="root evidence"): + search.run() + + +def test_late_draft_after_root_receipt_invalidation_is_charged_without_admission( + runtime, monkeypatch +): + control, adapter, snapshot, settings, root = runtime + dependency = tree.Dependency("root-proof", 0) + attempts = tuple(replace(a, dependencies=(dependency,)) for a in root.attempts) + by_id = {a.attempt_id: a for a in attempts} + initial = replace( + root, + attempts=attempts, + dependencies=(dependency,), + receipts=tuple(replace(r, attempt=by_id[r.attempt.attempt_id]) for r in root.receipts), + ) + search = Search(control, "campaign", "search", snapshot, settings, adapter) + search.start(initial, seed=2, collect=False) + original = control.settle + injected = False + + def invalidate_first(campaign, operation, receipt, **kwargs): + nonlocal injected + if receipt.get("candidate") and not injected: + injected = True + search.invalidate("root-proof") + return original(campaign, operation, receipt, **kwargs) + + monkeypatch.setattr(control, "settle", invalidate_first) + with pytest.raises(ValueError, match="root evidence"): + search.run() + assert injected + doc, record, state = search.read() + assert record["pending"] is None and record["selected"] is None + assert len(state.candidates) == 1 + assert all(op["status"] == "settled" for op in doc["operations"].values()) + assert record["rejected_drafts"] and record["spent_ms"] > 0 + + +def test_training_rejects_unretained_search_labels(runtime): + control, adapter, snapshot, settings, root = runtime + records = collect_group( + control, "campaign", adapter, root, snapshot, settings, group_id="natural", size=2, seed=12 + ) + altered = (records[0] | {"gain": 999}, records[1]) + with pytest.raises(ValueError, match="retained"): + train_group( + control, "campaign", altered, snapshot, snapshot, cap_ms=5000, learning_rate=0.01 + ) + + +@pytest.mark.parametrize("mode", ["reset", "carry"]) +def test_outer_loop_resume_reuses_trained_checkpoints_and_frozen_roots(runtime, mode): + from inferrsi.harness.search_campaign import learn + + control, adapter, snapshot, settings, root = runtime + settings = replace(settings, search_ms=10000, max_steps=3) + args = dict(rounds=2, group_size=2, training_cap_ms=5000, learning_rate=0.01, seed=8, mode=mode) + trained = learn(control, "campaign", adapter, root, snapshot, settings, **args) + before = control.read("campaign") + resumed = learn(control, "campaign", adapter, root, snapshot, settings, **args) + assert resumed == trained + assert control.read("campaign") == before + with pytest.raises(ValueError, match="starting conditions"): + learn(control, "campaign", adapter, root, snapshot, settings, **(args | {"seed": 9})) diff --git a/tests/test_work_control.py b/tests/test_work_control.py new file mode 100644 index 00000000..4e768423 --- /dev/null +++ b/tests/test_work_control.py @@ -0,0 +1,152 @@ +import time +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from inferrsi.harness.run_store import RunStore +from inferrsi.harness.work_control import WorkControl + + +@pytest.fixture +def control(tmp_path): + result = WorkControl(RunStore(tmp_path / "store", "controls")) + result.create( + "campaign", + {"fixed": "contract"}, + {"incumbent": "root"}, + cap_ms=1000, + final_reserve_ms=100, + deadline=time.time() + 60, + ) + return result + + +def claim(control, op="work", **changes): + args = dict(category="verify", cap_ms=200, domains=("device",), binding={"candidate": "c"}) + return control.claim("campaign", control.read("campaign")["revision"], op, **(args | changes)) + + +def test_atomic_resource_and_budget_reservation_across_controllers(control): + other = WorkControl(control.store) + other.create("other", {}, {}, cap_ms=1000, final_reserve_ms=100, deadline=time.time() + 60) + with ThreadPoolExecutor(2) as pool: + + def reserve(name): + try: + control.claim( + name, 0, name, category="verify", cap_ms=200, domains=("device",), binding={} + ) + return True + except ValueError: + return False + + assert sum(pool.map(reserve, ("campaign", "other"))) == 1 + + +def test_unknown_cleanup_or_usage_retains_reservation_and_never_relaunches(control): + claim(control) + control.launch("campaign", "work") + control.settle( + "campaign", "work", {"result": "unknown"}, actual_ms=None, cleanup_verified=False + ) + assert control.available(control.read("campaign")) == 700 + with pytest.raises(ValueError, match="already claimed"): + claim(control) + with pytest.raises(ValueError, match="leased"): + claim(control, "another") + reopened = WorkControl(RunStore(control.store.root, control.store.namespace)) + assert reopened.read("campaign")["operations"]["work"]["status"] == "unknown" + + +def test_receipt_settlement_and_incumbent_are_atomic_and_idempotent(control): + claim(control) + control.launch("campaign", "work") + + def update(state): + return state | {"incumbent": "child"} + + result = control.settle( + "campaign", + "work", + {"proof": "bound"}, + actual_ms=50, + cleanup_verified=True, + transition=update, + ) + assert result["state"]["incumbent"] == "child" + control.settle("campaign", "work", {"proof": "bound"}, actual_ms=50, cleanup_verified=True) + assert control.read("campaign")["spent_ms"] == 50 + with pytest.raises(ValueError, match="Conflicting"): + control.settle("campaign", "work", {}, actual_ms=50, cleanup_verified=True) + claim(control, "next") + + +def test_expired_claim_cannot_start_and_does_not_charge_unstarted_work(control, monkeypatch): + claim(control) + monkeypatch.setattr("inferrsi.harness.work_control.time.time", lambda: 1e12) + with pytest.raises(ValueError, match="expired"): + control.launch("campaign", "work") + result = control.read("campaign") + assert result["operations"]["work"]["status"] == "settled" + assert result["spent_ms"] == 0 + + +def test_cas_and_failed_transition_roll_back_accounting(control): + before = control.read("campaign") + control.update("campaign", before["revision"], lambda state: state | {"x": 1}) + with pytest.raises(ValueError, match="Stale"): + control.update("campaign", before["revision"], lambda state: {}) + claim(control) + control.launch("campaign", "work") + + def fail(state): + raise ValueError("invalid receipt") + + with pytest.raises(ValueError, match="invalid receipt"): + control.settle("campaign", "work", {}, actual_ms=20, cleanup_verified=True, transition=fail) + assert control.read("campaign")["spent_ms"] == 0 + + +def test_holdout_is_once_across_campaigns_even_after_interruption(control): + before = control.read("campaign") + control.claim_final("campaign", before["revision"], "heldout-data-v1", {"candidate": "c"}) + control.create("second", {}, {}, cap_ms=1000, final_reserve_ms=100, deadline=time.time() + 60) + with pytest.raises(ValueError, match="consumed"): + control.claim_final("second", 0, "heldout-data-v1", {"candidate": "other"}) + with pytest.raises(ValueError, match="frozen"): + claim(control) + control.finish_final("campaign", {"candidate": "c"}, {"verdict": "FAIL"}) + control.finish_final("campaign", {"candidate": "c"}, {"verdict": "FAIL"}) + with pytest.raises(ValueError, match="immutable"): + control.finish_final("campaign", {"candidate": "c"}, {"verdict": "PASS"}) + + +def test_final_reserve_and_unknown_work_cannot_be_spent(control): + with pytest.raises(ValueError, match="budget"): + claim(control, cap_ms=901) + claim(control) + with pytest.raises(ValueError, match="unresolved"): + control.claim_final("campaign", control.read("campaign")["revision"], "holdout", {}) + + +def test_over_budget_final_is_accounted_but_never_promoted(control): + control.claim_final("campaign", 0, "holdout", {"candidate": "c"}) + claim(control, final=True, category="final") + control.launch("campaign", "work") + control.settle("campaign", "work", {"verdict": "PASS"}, actual_ms=201, cleanup_verified=True) + assert control.read("campaign")["spent_ms"] == 201 + assert control.read("campaign")["blocked"] + with pytest.raises(ValueError, match="cannot promote"): + control.finish_final( + "campaign", + {"candidate": "c"}, + {"verdict": "PASS"}, + transition=lambda state: state | {"incumbent": "c"}, + ) + + +def test_read_only_control_does_not_create_or_change_tables(control): + from inferrsi.harness.run_store import RunStoreReader + + reader = WorkControl(RunStoreReader(control.store.root, control.store.namespace)) + assert reader.read("campaign") == control.read("campaign") diff --git a/tools/speculative_skill.py b/tools/speculative_skill.py new file mode 100644 index 00000000..548d5573 --- /dev/null +++ b/tools/speculative_skill.py @@ -0,0 +1,165 @@ +"""Explicit bounded-provider Skill comparator for the speculative-search Python API. + +This checkout tool composes the existing comparison authority. It never selects +a model/client implicitly, and does not turn fixed-output fixtures into live runs. +""" + +import argparse +import copy +import json +import os +import sys +import time +from pathlib import Path + +from inferrsi.harness import candidates +from inferrsi.harness.artifact_primitives import digest, sha256 +from inferrsi.harness.atomic_json import atomic_write_json +from inferrsi.harness.optimization_session import runtime_environment +from inferrsi.harness.scenarios.external import execute_trial +from inferrsi.knowledge.knowledge_operations import KnowledgeReader +from tools import skill_eval_compare as compare +from tools.skill_eval_improve import materialize + + +class Comparator: + """Bind a repo, original baseline and two disjoint externally declared splits. + + config contains comparison (the existing comparator config, including client), + allowed_paths, and splits. Each split has path, sha256, scope and task_count. + Holdout bytes are only opened by run(split='holdout'), after the caller's + WorkControl final claim. The controller must use that declared scope. + """ + + def __init__(self, repo, config, baseline_source): + self.repo = Path(repo).resolve() + self.config = copy.deepcopy(config) + if set(config) != {"comparison", "allowed_paths", "splits"}: + raise ValueError("Unknown or missing Skill comparator configuration") + comparison = config["comparison"] + if "client" not in comparison or comparison["mode"] != "update": + raise ValueError("Skill search requires an explicit bounded provider update config") + if set(config["splits"]) != {"development", "holdout"}: + raise ValueError("Separate development and holdout descriptors are required") + dev, final = (config["splits"][key] for key in ("development", "holdout")) + if dev["scope"] == final["scope"] or dev["sha256"] == final["sha256"]: + raise ValueError("Development and holdout must have distinct declared identities") + for split in (dev, final): + if ( + set(split) != {"path", "sha256", "scope", "task_count"} + or type(split["task_count"]) is not int + or split["task_count"] < 1 + or not Path(split["path"]).is_absolute() + ): + raise ValueError("Split requires an absolute source and frozen identity/count") + self.baseline = comparison["snapshots"]["baseline"]["git_revision"] + root = Path(baseline_source) + library = compare.snapshot(self.repo, self.baseline, comparison["skill_root"], root) + bundle = KnowledgeReader(skill_root=library).read("checkout:" + comparison["skill"]) + self.original = dict(bundle.content) + if set(config["allowed_paths"]) - self.original.keys(): + raise ValueError("Allowed edits must belong to the original Skill bundle") + self.identity = digest( + { + "config": config, + "baseline": self.baseline, + "resources": {k: digest(v.hex()) for k, v in self.original.items()}, + "controllers": {name: sha256(compare.ROOT / name) for name in compare.CONTROLLERS}, + } + ) + self._config_digest = digest(self.config) + self._controllers = {name: sha256(compare.ROOT / name) for name in compare.CONTROLLERS} + self.holdout_scope = "skill-data:" + final["sha256"] + + def export_root(self, destination): + destination = Path(destination) + destination.mkdir(parents=True, exist_ok=False) + for name, value in self.original.items(): + path = destination / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(value) + return destination + + def run(self, candidate, output, deadline, *, split): + if digest(self.config) != self._config_digest or any( + sha256(compare.ROOT / name) != expected for name, expected in self._controllers.items() + ): + raise ValueError("Frozen Skill configuration or controller changed") + files = candidates.members(candidate) + if set(files) != self.original.keys(): + raise ValueError("Skill candidate changed the frozen bundle membership") + edits = { + name: path.read_text(encoding="utf-8") + for name, path in files.items() + if path.read_bytes() != self.original[name] + } + if set(edits) - set(self.config["allowed_paths"]): + raise ValueError("Skill candidate changed a protected resource") + comparison = copy.deepcopy(self.config["comparison"]) + revisions, _ = materialize( + self.repo, + self.baseline, + comparison["skill_root"], + comparison["skill"], + {"inv": edits} if edits else {}, + self.config["allowed_paths"], + ) + descriptor = self.config["splits"][split] + tasks = Path(descriptor["path"]) + if sha256(tasks) != descriptor["sha256"]: + raise ValueError("Frozen Skill split changed") + cases = json.loads(tasks.read_text(encoding="utf-8")) + if len(cases) * comparison["limits"]["repeats"] != descriptor["task_count"]: + raise ValueError("Skill trial count differs from the frozen split") + if any(case.get("split") != split for case in cases): + raise ValueError("Skill task belongs to another declared split") + output.mkdir(parents=True, exist_ok=False) + comparison.update( + cases="tasks.json", + snapshots={ + "baseline": {"git_revision": self.baseline}, + "candidate": {"git_revision": revisions.get("inv", self.baseline)}, + }, + ) + atomic_write_json(output / "tasks.json", cases) + atomic_write_json(output / "config.json", comparison) + prepared = output / "comparison" + authorization = compare.prepare(output / "config.json", self.repo, prepared) + environment = runtime_environment() + environment["PYTHONPATH"] = str(compare.ROOT) + os.pathsep + environment["PYTHONPATH"] + command = ( + sys.executable, + "-m", + "tools.speculative_skill", + "--output", + str(prepared), + "--authorization", + authorization, + ) + record = execute_trial( + command, + compare.ROOT, + output / "execution", + time.monotonic() + max(0, deadline - time.time()), + None, + environment=environment, + include_cleanup=True, + ) + if sha256(prepared / "plan/freeze.json") != authorization: + raise ValueError("Skill comparison plan changed during execution") + report = compare.read_report(prepared) + atomic_write_json(output / "report.json", report) + return report, record["cleanup_verified"] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--authorization", required=True) + args = parser.parse_args() + compare.execute(args.output, args.authorization, require_request_authorization=True) + compare.report(args.output) + + +if __name__ == "__main__": + main() From 5ee418121af783bcd62289e23cffa702b7f560c2 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Wed, 18 Nov 2026 21:15:53 +0800 Subject: [PATCH 3/8] harness: retain evaluator identity across search execution Signed-off-by: amy-why-3459 --- src/inferrsi/apps/search_benchmarks.py | 6 ++++++ tests/test_search_benchmarks.py | 21 +++++++++++++++++++++ tools/speculative_skill.py | 11 +++++++++-- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/inferrsi/apps/search_benchmarks.py b/src/inferrsi/apps/search_benchmarks.py index eb7a1172..95b77931 100644 --- a/src/inferrsi/apps/search_benchmarks.py +++ b/src/inferrsi/apps/search_benchmarks.py @@ -73,6 +73,12 @@ def evaluate(self, candidate, check, output, deadline): environment=environment, include_cleanup=True, ) + if { + p.relative_to(self.frozen).as_posix(): sha256(p) + for p in self.frozen.rglob("*") + if p.is_file() + } != self.plan["frozen_files"]: + raise ValueError("Frozen benchmark resources changed during execution") assessed = protocol.assess(output, self.plan["evaluation_settings"], execution) if execution["cleanup_verified"] is not True: assessed = {"verdict": "INCONCLUSIVE", "score": None} diff --git a/tests/test_search_benchmarks.py b/tests/test_search_benchmarks.py index 5cc1ba6b..b2045f5d 100644 --- a/tests/test_search_benchmarks.py +++ b/tests/test_search_benchmarks.py @@ -78,6 +78,25 @@ def test_cli_requires_explicit_shared_store_and_read_commands_do_not_create_one( parser().parse_args(["search", "engine", "--session", str(tmp_path / "run")]) +def test_engine_rejects_frozen_resource_mutation_during_evaluation(tmp_path, monkeypatch): + evaluator = EngineEvaluator( + "qwen21-denoise-npu", + tmp_path / "frozen", + arguments=["--pairs", "1", "--samples", "1", "--warmups", "0"], + ) + source = tmp_path / "candidate" + source.mkdir() + (source / "candidate.py").write_text("unchanged candidate") + + def mutated_execution(*args, **kwargs): + (evaluator.frozen / "reference.py").write_text("changed reference") + return {"status": "completed", "returncode": 0, "cleanup_verified": True} + + monkeypatch.setattr("inferrsi.apps.search_benchmarks.execute_trial", mutated_execution) + with pytest.raises(ValueError, match="resources changed during"): + evaluator.evaluate(source, "evaluation", tmp_path / "output", time.time() + 1) + + def test_skill_comparator_preparation_does_not_access_holdout(tmp_path): from tests.fixtures.harness_improvement_demo import configuration from tools.speculative_skill import Comparator @@ -89,6 +108,7 @@ def test_skill_comparator_preparation_does_not_access_holdout(tmp_path): client={"explicit": "preparation-only-test"}, snapshots={"baseline": {"git_revision": config["baseline"]}}, ) + def descriptor(split): return { "path": str(tmp_path / (split + ".json")), @@ -96,6 +116,7 @@ def descriptor(split): "scope": split, "task_count": 3, } + bound = Comparator( repo, { diff --git a/tools/speculative_skill.py b/tools/speculative_skill.py index 548d5573..09cb18ed 100644 --- a/tools/speculative_skill.py +++ b/tools/speculative_skill.py @@ -59,16 +59,23 @@ def __init__(self, repo, config, baseline_source): self.original = dict(bundle.content) if set(config["allowed_paths"]) - self.original.keys(): raise ValueError("Allowed edits must belong to the original Skill bundle") + self._controllers = { + name: sha256(compare.ROOT / name) + for name in ( + *compare.CONTROLLERS, + "tools/speculative_skill.py", + "tools/skill_eval_improve.py", + ) + } self.identity = digest( { "config": config, "baseline": self.baseline, "resources": {k: digest(v.hex()) for k, v in self.original.items()}, - "controllers": {name: sha256(compare.ROOT / name) for name in compare.CONTROLLERS}, + "controllers": self._controllers, } ) self._config_digest = digest(self.config) - self._controllers = {name: sha256(compare.ROOT / name) for name in compare.CONTROLLERS} self.holdout_scope = "skill-data:" + final["sha256"] def export_root(self, destination): From bd2bfc432d76b2af8ddcae8b19b7936ac2cc6b73 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Wed, 18 Nov 2026 21:21:19 +0800 Subject: [PATCH 4/8] harness: bind GVO groups to identical initial observations Signed-off-by: amy-why-3459 --- docs/speculative-search.md | 5 +++-- src/inferrsi/harness/gvo.py | 5 ++++- src/inferrsi/harness/vats.py | 19 +++++++++++++++++++ tests/test_vats_runtime.py | 2 ++ 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/speculative-search.md b/docs/speculative-search.md index d7639bb5..b99e2c97 100644 --- a/docs/speculative-search.md +++ b/docs/speculative-search.md @@ -148,8 +148,9 @@ External process-liveness reconciliation needs a qualified authority. `finalize` consumes a unique scope in the shared store before final data access, freezes selection and closes search/training. Engine compares again with the -original baseline. For Skill, supply a **holdout-bound adapter** and the declared -holdout scope, never the development adapter. Exact bounded radix utility keeps +original baseline. For Skill, supply a **holdout-bound adapter** and +`Comparator.holdout_scope` (derived from the frozen data digest), never the +development adapter or a replacement label. Exact bounded radix utility keeps quality before token use; reduced tokens cannot offset a quality regression. After interruption, final consumes existing receipts only. Even an unstarted diff --git a/src/inferrsi/harness/gvo.py b/src/inferrsi/harness/gvo.py index 10e95096..f485040d 100644 --- a/src/inferrsi/harness/gvo.py +++ b/src/inferrsi/harness/gvo.py @@ -39,6 +39,7 @@ def examples(records, *, expense_weight=0.0, elapsed_weight=0.0): set(), ) returns = [] + initial_conditions = set() for record in records: if record["status"] != "complete" or record["pending"] is not None: raise ValueError("Incomplete searches cannot supply GVO returns") @@ -53,6 +54,7 @@ def examples(records, *, expense_weight=0.0, elapsed_weight=0.0): snapshots.add(record["snapshot"]) settings.add(record["settings"]) adapters.add(record["adapter"]) + initial_conditions.add(record["initial_conditions"]) returns.append( finite( record["gain"] @@ -62,7 +64,8 @@ def examples(records, *, expense_weight=0.0, elapsed_weight=0.0): ) ) if len(set(identities)) != len(identities) or any( - len(values) != 1 for values in (roots, contracts, snapshots, settings, adapters) + len(values) != 1 + for values in (roots, contracts, snapshots, settings, adapters, initial_conditions) ): raise ValueError("Search group must have distinct IDs and identical starting conditions") if any(record["forced"] for record in records): diff --git a/src/inferrsi/harness/vats.py b/src/inferrsi/harness/vats.py index bf79022f..6ed33bf5 100644 --- a/src/inferrsi/harness/vats.py +++ b/src/inferrsi/harness/vats.py @@ -235,6 +235,24 @@ def start(self, initial, *, seed, collect=True, forced=False): tree.accept(initial) if not forced and len(initial.candidates) != 1: raise ValueError("Independent natural searches must start from one qualified root") + + def observation(attempt): + return { + key: value + for key, value in asdict(attempt).items() + if key not in ("attempt_id", "search_id") + } + + initial_conditions = digest( + { + "candidates": [asdict(node) for node in initial.candidates], + "dependencies": [asdict(item) for item in initial.dependencies], + "attempts": [observation(a) for a in initial.attempts], + "receipts": [ + {**asdict(r), "attempt": observation(r.attempt)} for r in initial.receipts + ], + } + ) doc = self.control.read(self.campaign) def transition(state): @@ -257,6 +275,7 @@ def transition(state): "draft_attempts": 0, "active_action": None, "initial_gain": tree.verified_gain(initial), + "initial_conditions": initial_conditions, "selected": initial.root_id, } return state diff --git a/tests/test_vats_runtime.py b/tests/test_vats_runtime.py index bd3a793f..1873789a 100644 --- a/tests/test_vats_runtime.py +++ b/tests/test_vats_runtime.py @@ -161,6 +161,8 @@ def test_gvo_rejects_shared_tree_leaves_forced_and_changed_start_conditions(runt examples((records[0] | {"forced": True}, records[1])) with pytest.raises(ValueError, match="identical"): examples((records[0] | {"settings": "other"}, records[1])) + with pytest.raises(ValueError, match="identical"): + examples((records[0] | {"initial_conditions": "different-root-proof"}, records[1])) def test_snapshot_change_on_resume_is_rejected(runtime): From 96e3c50c54a9c30581492b8dc86f916e5c5fec43 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Wed, 18 Nov 2026 21:25:13 +0800 Subject: [PATCH 5/8] harness: inject search role operations across the model boundary Signed-off-by: amy-why-3459 --- docs/speculative-search.md | 3 +++ src/inferrsi/harness/gvo.py | 25 +++++++----------------- src/inferrsi/harness/search_adapter.py | 4 +--- src/inferrsi/harness/search_campaign.py | 3 +-- src/inferrsi/harness/vats.py | 9 +++------ src/inferrsi/model/search_roles.py | 26 +++++++++++++++++++++++++ 6 files changed, 41 insertions(+), 29 deletions(-) diff --git a/docs/speculative-search.md b/docs/speculative-search.md index b99e2c97..3c3e792b 100644 --- a/docs/speculative-search.md +++ b/docs/speculative-search.md @@ -123,6 +123,9 @@ Caller-authored unreceipted pair targets are rejected. These are real updates to a constrained CPU operator model. They are not LLM finetuning, prompt summarization or evidence of pretrained-model improvement. +Apps injects the role snapshot's `choose`, `predict`, `train`, `restore` and +`document` operations. Harness retains traces and controls when they may run; +it has no import dependency on Model. ## Accounting, recovery and final diff --git a/src/inferrsi/harness/gvo.py b/src/inferrsi/harness/gvo.py index f485040d..71490cc0 100644 --- a/src/inferrsi/harness/gvo.py +++ b/src/inferrsi/harness/gvo.py @@ -5,23 +5,14 @@ import uuid from dataclasses import replace -from inferrsi.model.search_roles import Choice, finite, update - from .artifact_primitives import digest from .search_records import decode -def choice_from_document(raw): - return Choice( - **( - raw - | { - "options": tuple(raw["options"]), - "features": tuple(tuple(row) for row in raw["features"]), - "probabilities": tuple(raw["probabilities"]), - } - ) - ) +def finite(value): + if type(value) not in (int, float) or not math.isfinite(value): + raise ValueError("Search returns require finite recorded observations") + return value def examples(records, *, expense_weight=0.0, elapsed_weight=0.0): @@ -85,9 +76,8 @@ def examples(records, *, expense_weight=0.0, elapsed_weight=0.0): if step["forced"]: raise ValueError("Forced action leaked into a natural trajectory") for raw in (step["choice"], *step["draft_choices"]): - choice = choice_from_document(raw) - if choice.sampled: - actors.append((choice, advantage)) + if raw["sampled"]: + actors.append((raw, advantage)) return actors, values, {"search_ids": identities, "returns": returns, "advantages": advantages} @@ -183,8 +173,7 @@ def train_group( values.extend( suffix_examples(record, expense_weight=expense_weight, elapsed_weight=elapsed_weight) ) - trained = update( - snapshot, + trained = snapshot.train( reference, actors, values, diff --git a/src/inferrsi/harness/search_adapter.py b/src/inferrsi/harness/search_adapter.py index 4896f8a0..8375337f 100644 --- a/src/inferrsi/harness/search_adapter.py +++ b/src/inferrsi/harness/search_adapter.py @@ -12,7 +12,6 @@ from pathlib import Path from inferrsi.contracts.claim import Verdict -from inferrsi.model.search_roles import choose from . import candidates from . import search_tree as tree @@ -131,8 +130,7 @@ def draft(self, parent, state, snapshot, rng, operation, deadline): vectors = [ tuple(float(i == index) for i in range(len(self.edits))) for index, _ in legal ] - choice = choose( - snapshot, + choice = snapshot.choose( "drafter", [edit.token for _, edit in legal], vectors, diff --git a/src/inferrsi/harness/search_campaign.py b/src/inferrsi/harness/search_campaign.py index fea8bce9..96897c51 100644 --- a/src/inferrsi/harness/search_campaign.py +++ b/src/inferrsi/harness/search_campaign.py @@ -4,7 +4,6 @@ from dataclasses import asdict, replace from inferrsi.contracts.claim import Verdict -from inferrsi.model.search_roles import Snapshot from . import search_tree as tree from .artifact_primitives import digest @@ -215,7 +214,7 @@ def learn( or digest(op["binding"]["training"]) != digest(expected) ): raise ValueError("Frozen outer-loop training configuration changed") - snapshot = Snapshot(**op["receipt"]["checkpoint"]) + snapshot = snapshot.restore(op["receipt"]["checkpoint"]) else: snapshot = train_group( control, diff --git a/src/inferrsi/harness/vats.py b/src/inferrsi/harness/vats.py index 6ed33bf5..914ca415 100644 --- a/src/inferrsi/harness/vats.py +++ b/src/inferrsi/harness/vats.py @@ -12,8 +12,6 @@ from dataclasses import asdict, dataclass, replace from itertools import combinations -from inferrsi.model.search_roles import choose, dot - from . import search_tree as tree from .artifact_primitives import digest from .search_records import decode, encode @@ -140,8 +138,8 @@ def menu(state, jobs, available, settings, snapshot): continue trial = allowed[frozenset(union)] queries += 2 - if dot(snapshot.scorer, features(state, trial, available, settings)) > dot( - snapshot.scorer, features(state, left, available, settings) + if snapshot.predict(features(state, trial, available, settings)) > snapshot.predict( + features(state, left, available, settings) ): extra.append(trial) if not settings.bundled: @@ -526,8 +524,7 @@ def run(self, *, forced_action=None): features(current, action, remaining, self.settings) for action in actions ) rng = random.Random(f"{search['seed']}:policy:{len(search['trace'])}") - choice = choose( - self.snapshot, + choice = self.snapshot.choose( "policy", [action.key for action in actions], vectors, diff --git a/src/inferrsi/model/search_roles.py b/src/inferrsi/model/search_roles.py index 3e5df28c..e0a8e950 100644 --- a/src/inferrsi/model/search_roles.py +++ b/src/inferrsi/model/search_roles.py @@ -61,6 +61,32 @@ def document(self): "scorer": list(self.scorer), } + def choose(self, *args, **kwargs): + """Injected role operation; Harness does not import this Model module.""" + return choose(self, *args, **kwargs) + + def predict(self, features): + return dot(self.scorer, features) + + def restore(self, document): + return type(self)(**document) + + def train(self, reference, actors, values, pairs, **settings): + restored = [] + for raw, advantage in actors: + choice = Choice( + **( + raw + | { + "options": tuple(raw["options"]), + "features": tuple(tuple(row) for row in raw["features"]), + "probabilities": tuple(raw["probabilities"]), + } + ) + ) + restored.append((choice, advantage)) + return update(self, reference, restored, values, pairs, **settings) + @dataclass(frozen=True) class Choice: From bb6ecbd5a6888f43345ba5376f71f2c6e55f0c25 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Wed, 18 Nov 2026 21:42:11 +0800 Subject: [PATCH 6/8] apps: isolate benchmark loading and use bounded protocol processes Signed-off-by: amy-why-3459 --- src/inferrsi/apps/search_benchmarks.py | 21 +++++++++++++++++---- src/inferrsi/apps/speculative_search.py | 4 ++-- src/inferrsi/harness/task_protocol.py | 18 ++++++++++++++---- tests/test_task_protocol_child.py | 16 ++++++++++++++++ 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/src/inferrsi/apps/search_benchmarks.py b/src/inferrsi/apps/search_benchmarks.py index 95b77931..b89442d1 100644 --- a/src/inferrsi/apps/search_benchmarks.py +++ b/src/inferrsi/apps/search_benchmarks.py @@ -49,9 +49,14 @@ def evaluate(self, candidate, check, output, deadline): raise ValueError("Frozen benchmark resources changed") output.mkdir(parents=True) candidates.retain(candidate / "candidate.py", output / "candidate.py") - protocol = task_protocol.protocol(self.frozen) - args = protocol.arguments( - output / "candidate.py", output / "measurement", self.plan["evaluation_settings"] + args = task_protocol.run( + self.frozen, + self.plan["frozen_files"]["optimization.py"], + "arguments", + deadline=time.monotonic() + max(0, deadline - time.time()), + candidate=output / "candidate.py", + output=output / "measurement", + settings=self.plan["evaluation_settings"], ) from inferrsi.harness import optimization_session @@ -79,7 +84,15 @@ def evaluate(self, candidate, check, output, deadline): if p.is_file() } != self.plan["frozen_files"]: raise ValueError("Frozen benchmark resources changed during execution") - assessed = protocol.assess(output, self.plan["evaluation_settings"], execution) + assessed = task_protocol.run( + self.frozen, + self.plan["frozen_files"]["optimization.py"], + "assess", + deadline=time.monotonic() + max(0, deadline - time.time()), + directory=output, + settings=self.plan["evaluation_settings"], + execution=execution, + ) if execution["cleanup_verified"] is not True: assessed = {"verdict": "INCONCLUSIVE", "score": None} atomic_write_json(output / "assessment.json", assessed) diff --git a/src/inferrsi/apps/speculative_search.py b/src/inferrsi/apps/speculative_search.py index 318939f0..01deb8b3 100644 --- a/src/inferrsi/apps/speculative_search.py +++ b/src/inferrsi/apps/speculative_search.py @@ -17,8 +17,6 @@ from inferrsi.harness.work_control import WorkControl from inferrsi.model.search_roles import Snapshot -from .search_benchmarks import EngineEvaluator - def add_arguments(commands): search = commands.add_parser("search", help="Bounded SpecRSI search and role training") @@ -80,6 +78,8 @@ def command(args): if args.search_action == "export": atomic_write_json(args.output, result) return result + from .search_benchmarks import EngineEvaluator + if root.exists(): raise ValueError("Use a new session directory; status/export never restart work") if args.groups < 2 or args.rounds < 1: diff --git a/src/inferrsi/harness/task_protocol.py b/src/inferrsi/harness/task_protocol.py index 340f1218..792b5564 100644 --- a/src/inferrsi/harness/task_protocol.py +++ b/src/inferrsi/harness/task_protocol.py @@ -7,6 +7,7 @@ Windows Job, and returns one strict JSON envelope (#412 R3). """ +import math import os import shutil import sys @@ -85,15 +86,23 @@ def _response_problem(response, operation: str) -> str | None: return None -def run(root: Path, protocol_sha256: str, operation: str, **arguments): +def run(root: Path, protocol_sha256: str, operation: str, *, deadline=None, **arguments): """Run one operation of ``root/optimization.py`` whose bytes have ``protocol_sha256``. Path arguments must be absolute. The child's stderr tail is copied to this process's stderr after every operation. A protocol ``SystemExit`` is raised again with its exit code; every other failure raises ``TaskProtocolError``. Descendants that the - protocol leaves running are stopped with the child. + protocol leaves running are stopped with the child. An optional absolute + monotonic deadline further bounds the shared protocol timeout. """ path = Path(root).resolve() / "optimization.py" + timeout = PROTOCOL_TIMEOUT_SECONDS + if deadline is not None: + if type(deadline) not in (int, float) or not math.isfinite(deadline): + raise TaskProtocolError("request", "deadline must be finite monotonic time") + timeout = min(timeout, deadline - time.monotonic()) + if timeout <= 0: + raise TaskProtocolError("timeout", "deadline expired before protocol execution") if sha256(path) != protocol_sha256: raise TaskProtocolError("identity", "protocol bytes differ from the frozen digest") request = {"schema_version": protocol_child.SCHEMA_VERSION, "operation": operation, @@ -113,7 +122,8 @@ def run(root: Path, protocol_sha256: str, operation: str, **arguments): # cannot land among the frozen resources. record = local_process._supervise( argv, work, dict(os.environ), work / "stdout", work / "stderr", - time.monotonic() + PROTOCOL_TIMEOUT_SECONDS) + min(time.monotonic() + timeout, deadline) if deadline is not None + else time.monotonic() + timeout) stderr = _tail(work / "stderr") if (work / "stderr").is_file() else "" # Diagnostics contract: after every operation, whatever its result, the last # STDERR_TAIL_BYTES of the child's stderr (protocol prints included) are copied to @@ -125,7 +135,7 @@ def run(root: Path, protocol_sha256: str, operation: str, **arguments): raise KeyboardInterrupt if record["status"] == "timeout": raise TaskProtocolError( - "timeout", f"{operation} exceeded {PROTOCOL_TIMEOUT_SECONDS:g} s; {stderr}") + "timeout", f"{operation} exceeded {timeout:g} s; {stderr}") if record["status"] not in ("completed", "failed"): raise TaskProtocolError("launch", f"{record['status']}: {record['error']}") if record["returncode"] != 0: diff --git a/tests/test_task_protocol_child.py b/tests/test_task_protocol_child.py index 0446015c..973d1191 100644 --- a/tests/test_task_protocol_child.py +++ b/tests/test_task_protocol_child.py @@ -219,6 +219,22 @@ def test_deadline_stops_the_child_and_its_descendants(tmp_path, monkeypatch, cap assert not marker.exists() +def test_campaign_deadline_bounds_the_protocol_timeout(tmp_path): + digest = write_protocol( + tmp_path, "import time\ndef validate_arguments(a):\n time.sleep(60)\n") + started = time.monotonic() + error = failure(tmp_path, digest, deadline=started + 1.0, arguments=[]) + assert error.kind == "timeout" + assert time.monotonic() - started < 10 + + +def test_expired_campaign_deadline_never_starts_protocol(tmp_path, monkeypatch): + digest = write_protocol(tmp_path, "def validate_arguments(a):\n return {}\n") + monkeypatch.setattr(task_protocol.local_process, "_supervise", + lambda *a, **k: pytest.fail("expired protocol must not start")) + assert failure(tmp_path, digest, deadline=0, arguments=[]).kind == "timeout" + + @pytest.mark.parametrize("body, raises", [ (" return {}\n", False), (" raise ValueError('rejected')\n", True), From dc7292a3cd3d624794ed280a0d677d20e03fff29 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Wed, 18 Nov 2026 22:02:39 +0800 Subject: [PATCH 7/8] harness: reconcile unstarted search work and retain known overruns Recover conclusively unstarted work without adapter execution, keep unknown launches blocked, and reserve known usage until cleanup is reconciled. Reject legacy searches without retained initial evidence and cover the reviewed planner, role, identity and deadline boundaries. Agent-assisted-by: Codex (implementation) Signed-off-by: amy-why-3459 --- docs/speculative-search.md | 18 +++- src/inferrsi/harness/gvo.py | 8 +- src/inferrsi/harness/search_records.py | 8 ++ src/inferrsi/harness/vats.py | 41 +++++++- src/inferrsi/harness/work_control.py | 22 +++- tests/test_search_benchmarks.py | 35 +++++++ tests/test_search_roles.py | 19 ++++ tests/test_task_protocol_child.py | 8 ++ tests/test_vats_runtime.py | 135 +++++++++++++++++++++++++ tests/test_verification_plans.py | 14 +++ tests/test_work_control.py | 33 ++++++ 11 files changed, 329 insertions(+), 12 deletions(-) diff --git a/docs/speculative-search.md b/docs/speculative-search.md index 3c3e792b..bf7020c8 100644 --- a/docs/speculative-search.md +++ b/docs/speculative-search.md @@ -75,6 +75,12 @@ Optional refinement adds unions without removing base completions. An undersized menu cap fails explicitly. Collection samples Policy; deployment uses argmax. Drafter edits remain sampled from the supplied permitted vocabulary. +Experimental records predating `initial_conditions` remain readable/exportable +through WorkControl, but cannot resume or enter GVO training. Start fresh searches +from qualified evidence; there is no migration that guesses a historical starting +tree from a later tree. Receiving search/attempt IDs may differ after an explicit +root import; actual execution identities, evidence and dependency epochs must agree. + The chosen work block is persisted before execution. Each edit waits for its parent artifact, without waiting for parent quality. Each check binds candidate, contract, attempt, execution, reservation and domain. Settlement commits elapsed @@ -138,7 +144,10 @@ use the **same shared store**. Launch intent is durable before dispatch; an ambiguous crash never authorizes a second launch. Known elapsed time and cleanup release unused reservation. Unknown usage/cleanup retains the entire reservation and domain. Over-cap work -is charged and blocks promotion. Absolute deadlines include queueing and cleanup. +blocks further claims immediately, even with unknown cleanup; the retained hold +is at least the largest known usage. Later reconciliation releases the domain +and charges the final usage once, never below that known lower bound. Absolute +deadlines include queueing and cleanup. In-process bounded planning, editing and training are checked after return; they are not forcibly preempted like subprocesses. Preflight freezing and initial setup are outside this campaign meter, a remaining lifecycle-cost limitation. @@ -149,6 +158,13 @@ checks retained candidate bytes and never launches new work. Lost preparation, training or copy receipts remain unresolved; no guessed-cost retry is available. External process-liveness reconciliation needs a qualified authority. +An expired launch recorded as `settled/not_started` is conclusive control evidence +that dispatch never occurred. Search recovery clears its pending work and ends +the current block; a registered verification attempt gets a control-only +`INCONCLUSIVE` receipt, with no objective or evaluator PASS. No execution is +charged. A search whose limit has expired can then return its qualified incumbent. +This exception does not apply to ambiguous launch-intent crashes. + `finalize` consumes a unique scope in the shared store before final data access, freezes selection and closes search/training. Engine compares again with the original baseline. For Skill, supply a **holdout-bound adapter** and diff --git a/src/inferrsi/harness/gvo.py b/src/inferrsi/harness/gvo.py index 71490cc0..1fa86503 100644 --- a/src/inferrsi/harness/gvo.py +++ b/src/inferrsi/harness/gvo.py @@ -6,7 +6,7 @@ from dataclasses import replace from .artifact_primitives import digest -from .search_records import decode +from .search_records import decode, initial_conditions def finite(value): @@ -30,7 +30,7 @@ def examples(records, *, expense_weight=0.0, elapsed_weight=0.0): set(), ) returns = [] - initial_conditions = set() + starts = set() for record in records: if record["status"] != "complete" or record["pending"] is not None: raise ValueError("Incomplete searches cannot supply GVO returns") @@ -45,7 +45,7 @@ def examples(records, *, expense_weight=0.0, elapsed_weight=0.0): snapshots.add(record["snapshot"]) settings.add(record["settings"]) adapters.add(record["adapter"]) - initial_conditions.add(record["initial_conditions"]) + starts.add(initial_conditions(record)) returns.append( finite( record["gain"] @@ -56,7 +56,7 @@ def examples(records, *, expense_weight=0.0, elapsed_weight=0.0): ) if len(set(identities)) != len(identities) or any( len(values) != 1 - for values in (roots, contracts, snapshots, settings, adapters, initial_conditions) + for values in (roots, contracts, snapshots, settings, adapters, starts) ): raise ValueError("Search group must have distinct IDs and identical starting conditions") if any(record["forced"] for record in records): diff --git a/src/inferrsi/harness/search_records.py b/src/inferrsi/harness/search_records.py index 8377a358..343ad49d 100644 --- a/src/inferrsi/harness/search_records.py +++ b/src/inferrsi/harness/search_records.py @@ -11,6 +11,14 @@ def encode(state): return asdict(state) +def initial_conditions(record): + """Reject legacy records whose actual starting evidence was never retained.""" + value = record.get("initial_conditions") + if not isinstance(value, str) or not value: + raise ValueError("Legacy search lacks initial evidence identity; start a fresh search") + return value + + def decode(raw): def dependencies(values): return tuple(tree.Dependency(**value) for value in values) diff --git a/src/inferrsi/harness/vats.py b/src/inferrsi/harness/vats.py index 914ca415..ab8b67ac 100644 --- a/src/inferrsi/harness/vats.py +++ b/src/inferrsi/harness/vats.py @@ -14,7 +14,7 @@ from . import search_tree as tree from .artifact_primitives import digest -from .search_records import decode, encode +from .search_records import decode, encode, initial_conditions from .verification_plans import completion_menu @@ -216,6 +216,7 @@ def __init__(self, control, campaign, search_id, snapshot, settings, adapter): def read(self): document = self.control.read(self.campaign) search = document["state"]["searches"][self.search_id] + initial_conditions(search) if ( search["snapshot"] != self.snapshot_id or search["settings"] != self.settings_id @@ -371,11 +372,45 @@ def transition(state): self.control.update(self.campaign, doc["revision"], transition) def recover(self): - _, search, _ = self.read() + doc, search, _ = self.read() pending = search["pending"] if pending is None: return - # Receipt-only recovery. Never call draft/verify from this path. + operation = doc["operations"][pending["operation"]] + if operation["status"] == "settled" and operation["receipt"] == {"not_started": "deadline"}: + # launch() conclusively refused dispatch. There can be no adapter + # receipt: settle the Attempt as control-only INCONCLUSIVE, and + # abandon the remaining block without inventing check evidence. + def not_started(state): + from inferrsi.contracts.claim import Verdict + + record = state["searches"][self.search_id] + current = decode(record["tree"]) + if pending["kind"] == "verify": + bound = next( + a for a in current.attempts if a.attempt_id == pending["operation"] + ) + current = tree.record_receipt( + current, + tree.Receipt(bound, Verdict.INCONCLUSIVE, digest(operation)), + expected_revision=current.revision, + ) + record.update( + tree=encode(current), + pending=None, + active_action=None, + block_stop_reason="launch_deadline", + ) + try: + record["selected"] = tree.accept(current).candidate_id + except ValueError: + record.update(status="blocked", selected=None) + return state + + self.control.update(self.campaign, doc["revision"], not_started) + return + # All uncertain launches still require an adapter receipt. Never call + # draft/verify from recovery, including after launch-intent crashes. result = self.adapter.recover(pending["operation"], pending) if result is None: raise ValueError("Unknown execution: retain reservation and do not relaunch") diff --git a/src/inferrsi/harness/work_control.py b/src/inferrsi/harness/work_control.py index 3a0993c4..97ace875 100644 --- a/src/inferrsi/harness/work_control.py +++ b/src/inferrsi/harness/work_control.py @@ -2,7 +2,8 @@ This is a controller for trusted host operations, not an executor or sandbox. All cooperating controllers must share the same RunStore for resource domains -and holdout scopes to be exclusive. Unknown work retains its entire reservation. +and holdout scopes to be exclusive. Unknown work retains at least its entire +reservation and any larger known usage lower bound. """ import json @@ -101,7 +102,11 @@ def read(self, campaign): @staticmethod def available(body, *, final=False): - held = sum(op["cap_ms"] for op in body["operations"].values() if op["status"] != "settled") + held = sum( + max(op["cap_ms"], op.get("known_ms", 0)) + for op in body["operations"].values() + if op["status"] != "settled" + ) return body["cap_ms"] - body["spent_ms"] - held - (0 if final else body["final_reserve_ms"]) def update(self, campaign, revision, transition): @@ -209,6 +214,16 @@ def settle(self, campaign, operation, receipt, *, actual_ms, cleanup_verified, t return if op["status"] not in ("launched", "unknown"): raise ValueError("Cannot settle an unlaunched operation") + known_ms = max(op.get("known_ms", 0), actual_ms or 0) + if cleanup_verified is True and actual_ms is not None and actual_ms < known_ms: + raise ValueError("Final usage cannot be below previously known usage") + op["known_ms"] = known_ms + executable_ms = min(op["cap_ms"], (op["deadline"] - op["started_at"]) * 1000) + # Cleanup uncertainty cannot erase an observed overrun. Keep the + # domain and conservative hold until reconciliation, blocking any + # further work immediately, including work on other domains. + if known_ms > executable_ms: + body["blocked"] = True op["settlement"] = settlement if actual_ms is None or cleanup_verified is not True: op["status"] = "unknown" @@ -218,8 +233,7 @@ def settle(self, campaign, operation, receipt, *, actual_ms, cleanup_verified, t else: op.update(status="settled", actual_ms=actual_ms, receipt=receipt) body["spent_ms"] += actual_ms - executable_ms = min(op["cap_ms"], (op["deadline"] - op["started_at"]) * 1000) - if actual_ms > executable_ms or body["spent_ms"] > body["cap_ms"]: + if body["spent_ms"] > body["cap_ms"]: body["blocked"] = True if transition is not None and not body["blocked"]: body["state"] = transition(body["state"]) diff --git a/tests/test_search_benchmarks.py b/tests/test_search_benchmarks.py index b2045f5d..bf957bb1 100644 --- a/tests/test_search_benchmarks.py +++ b/tests/test_search_benchmarks.py @@ -97,6 +97,41 @@ def mutated_execution(*args, **kwargs): evaluator.evaluate(source, "evaluation", tmp_path / "output", time.time() + 1) +def test_engine_protocol_stages_share_the_original_deadline(tmp_path, monkeypatch): + evaluator = EngineEvaluator( + "qwen21-denoise-npu", tmp_path / "frozen", + arguments=["--pairs", "1", "--samples", "1", "--warmups", "0"], + ) + source = tmp_path / "candidate" + source.mkdir() + (source / "candidate.py").write_text("candidate") + clock = [100.0] + observed = [] + monkeypatch.setattr("inferrsi.apps.search_benchmarks.time.time", lambda: clock[0]) + monkeypatch.setattr("inferrsi.apps.search_benchmarks.time.monotonic", lambda: clock[0] + 1000) + + def protocol(root, digest, operation, *, deadline, **kwargs): + observed.append((operation, deadline, deadline - (clock[0] + 1000))) + if operation == "arguments": + clock[0] += 2 + return [] + clock[0] += 1 + return {"verdict": "INCONCLUSIVE", "score": None} + + def execute(command, cwd, output, deadline, *args, **kwargs): + observed.append(("execution", deadline, deadline - (clock[0] + 1000))) + output.mkdir(parents=True) + (output / "execution.json").write_text("{}") + clock[0] += 3 + return {"status": "completed", "returncode": 0, "cleanup_verified": True} + + monkeypatch.setattr("inferrsi.apps.search_benchmarks.task_protocol.run", protocol) + monkeypatch.setattr("inferrsi.apps.search_benchmarks.execute_trial", execute) + evaluator.evaluate(source, "evaluation", tmp_path / "output", 110.0) + assert observed == [("arguments", 1110.0, 10.0), ("execution", 1110.0, 8.0), + ("assess", 1110.0, 5.0)] + + def test_skill_comparator_preparation_does_not_access_holdout(tmp_path): from tests.fixtures.harness_improvement_demo import configuration from tools.speculative_skill import Comparator diff --git a/tests/test_search_roles.py b/tests/test_search_roles.py index 3378e7be..1ff9219a 100644 --- a/tests/test_search_roles.py +++ b/tests/test_search_roles.py @@ -1,4 +1,6 @@ +import json import random +from dataclasses import asdict import pytest @@ -69,3 +71,20 @@ def test_clipped_actor_does_not_reinforce_outside_positive_ratio_bound(): ) weights = (10.0, -10.0) if choice.selected == 0 else (-10.0, 10.0) assert actor_gradient(weights, model.policy, choice, 1.0, 0.2, 0.0) == (0.0, 0.0) + + +def test_injected_roles_preserve_serialized_behavior_and_checkpoint_operations(): + model = Snapshot((0.2, -0.1), (0.3, -0.2), (0.5, -0.25), "initial") + vectors = ((1.0, 0.0), (0.0, 1.0)) + options = dict(context="observed", collect=True, temperature=0.8, exploration=0.1) + choice = model.choose("policy", ("a", "b"), vectors, rng=random.Random(7), **options) + assert choice == choose(model, "policy", ("a", "b"), vectors, + rng=random.Random(7), **options) + actors = [(json.loads(json.dumps(asdict(choice))), 2.0)] + values = [(vectors[0], 4.0)] + trained = model.train(model, actors, values, (), learning_rate=0.1) + assert trained == update(model, model, [(choice, 2.0)], values, learning_rate=0.1) + restored = model.restore(json.loads(json.dumps(trained.document()))) + assert restored == trained and restored != model + assert model.predict((2.0, 4.0)) == 0.0 + assert restored.predict(vectors[0]) == trained.scorer[0] diff --git a/tests/test_task_protocol_child.py b/tests/test_task_protocol_child.py index 973d1191..c7689803 100644 --- a/tests/test_task_protocol_child.py +++ b/tests/test_task_protocol_child.py @@ -235,6 +235,14 @@ def test_expired_campaign_deadline_never_starts_protocol(tmp_path, monkeypatch): assert failure(tmp_path, digest, deadline=0, arguments=[]).kind == "timeout" +@pytest.mark.parametrize("deadline", [float("nan"), float("inf"), -float("inf"), True, "soon"]) +def test_invalid_campaign_deadline_never_starts_protocol(tmp_path, monkeypatch, deadline): + digest = write_protocol(tmp_path, "def validate_arguments(a):\n return {}\n") + monkeypatch.setattr(task_protocol.local_process, "_supervise", + lambda *a, **k: pytest.fail("invalid deadline must not start")) + assert failure(tmp_path, digest, deadline=deadline, arguments=[]).kind == "request" + + @pytest.mark.parametrize("body, raises", [ (" return {}\n", False), (" raise ValueError('rejected')\n", True), diff --git a/tests/test_vats_runtime.py b/tests/test_vats_runtime.py index 1873789a..522667cf 100644 --- a/tests/test_vats_runtime.py +++ b/tests/test_vats_runtime.py @@ -150,6 +150,83 @@ def crash(campaign, operation, receipt, **kwargs): assert len(list((adapter.root / "candidates").iterdir())) == before +@pytest.mark.parametrize("kind", ["draft", "verify"]) +def test_expired_launch_recovers_unstarted_block_and_finishes_with_root(runtime, monkeypatch, kind): + control, adapter, snapshot, settings, root = runtime + search = Search(control, "campaign", root.contract.search_id, snapshot, settings, adapter) + search.start(root, seed=2, collect=False) + launch = control.launch + interrupted = {} + + def expire(campaign, operation): + doc = control.read(campaign) + if doc["operations"][operation]["category"] == kind: + interrupted.update(operation=operation, spent_ms=doc["spent_ms"]) + monkeypatch.setattr( + "inferrsi.harness.work_control.time.time", lambda: doc["deadline"] + 1 + ) + return launch(campaign, operation) + + monkeypatch.setattr(control, "launch", expire) + with pytest.raises(ValueError, match="expired before launch"): + search.run() + assert interrupted + operation = interrupted["operation"] + op = control.read("campaign")["operations"][operation] + assert op["receipt"] == {"not_started": "deadline"} + assert not (adapter.root / "executions" / operation).exists() + + def no_adapter_recovery(*args): + pytest.fail("Conclusive no-launch recovery must use the durable control record") + + monkeypatch.setattr(adapter, "recover", no_adapter_recovery) + reopened = WorkControl(RunStore(control.store.root, control.store.namespace)) + resumed = Search(reopened, "campaign", root.contract.search_id, snapshot, settings, adapter) + resumed.recover() + after = reopened.read("campaign") + resumed.recover() + assert reopened.read("campaign") == after + record = resumed.run() + assert record["status"] == "complete" and record["stop_reason"] == "limit" + assert record["selected"] == root.root_id + assert record["pending"] is None and record["active_action"] is None + assert record["block_stop_reason"] == "launch_deadline" + assert reopened.read("campaign")["spent_ms"] == interrupted["spent_ms"] + if kind == "verify": + state = decode(record["tree"]) + receipt = next(r for r in state.receipts if r.attempt.attempt_id == operation) + assert receipt.verdict.value == "INCONCLUSIVE" and receipt.objective is None + assert receipt.attempt.check_id in tree.missing_checks(state, receipt.attempt.candidate_id) + with reopened.store._connection() as db: + lease = db.execute("SELECT 1 FROM work_domains WHERE operation=?", (operation,)).fetchone() + assert lease is None + + +@pytest.mark.parametrize("kind", ["draft", "verify"]) +def test_unknown_launch_is_not_treated_as_conclusively_unstarted(runtime, monkeypatch, kind): + control, adapter, snapshot, settings, root = runtime + search = Search(control, "campaign", root.contract.search_id, snapshot, settings, adapter) + search.start(root, seed=2, collect=False) + launch = control.launch + + def crash_after_intent(campaign, operation): + result = launch(campaign, operation) + if result["category"] == kind: + raise RuntimeError("lost after launch intent") + return result + + monkeypatch.setattr(control, "launch", crash_after_intent) + with pytest.raises(RuntimeError, match="lost after launch intent"): + search.run() + before = control.read("campaign") + with pytest.raises(ValueError, match="Unknown execution"): + search.recover() + assert control.read("campaign") == before + assert search.read()[1]["pending"] is not None + with pytest.raises(ValueError, match="unsettled"): + search.finish("limit") + + def test_gvo_rejects_shared_tree_leaves_forced_and_changed_start_conditions(runtime): control, adapter, snapshot, settings, root = runtime records = collect_group( @@ -163,6 +240,64 @@ def test_gvo_rejects_shared_tree_leaves_forced_and_changed_start_conditions(runt examples((records[0] | {"settings": "other"}, records[1])) with pytest.raises(ValueError, match="identical"): examples((records[0] | {"initial_conditions": "different-root-proof"}, records[1])) + legacy = {k: v for k, v in records[0].items() if k != "initial_conditions"} + with pytest.raises(ValueError, match="Legacy search"): + examples((legacy, records[1])) + + +def test_legacy_search_can_be_exported_but_cannot_resume_without_initial_evidence(runtime): + control, adapter, snapshot, settings, root = runtime + search = Search(control, "campaign", root.contract.search_id, snapshot, settings, adapter) + search.start(root, seed=0) + + def legacy(state): + state["searches"][search.search_id].pop("initial_conditions") + return state + + before = control.update("campaign", control.read("campaign")["revision"], legacy) + with pytest.raises(ValueError, match="Legacy search"): + search.run() + assert control.read("campaign") == before + + +@pytest.mark.parametrize("change", ["receipt", "execution", "epoch"]) +def test_initial_evidence_digest_ignores_only_receiving_ids(runtime, change): + from inferrsi.harness.search_campaign import import_qualified_root + + control, adapter, snapshot, settings, root = runtime + first = Search(control, "campaign", root.contract.search_id, snapshot, settings, adapter) + first.start(root, seed=0) + imported = import_qualified_root(root, replace(root.contract, search_id="same-proof")) + second = Search(control, "campaign", "same-proof", snapshot, settings, adapter) + second.start(imported, seed=1) + assert first.read()[1]["initial_conditions"] == second.read()[1]["initial_conditions"] + assert examples((first.finish("test"), second.finish("test")))[2]["returns"] == [0, 0] + + changed = import_qualified_root(root, replace(root.contract, search_id="changed-proof")) + receipt = changed.receipts[0] + if change == "receipt": + receipt = replace(receipt, evidence_digest=digest("different real observation")) + elif change == "execution": + receipt = replace(receipt, attempt=replace(receipt.attempt, execution_key="another-run")) + else: + dependency = tree.Dependency("proof-epoch", 1) + receipt = replace(receipt, attempt=replace(receipt.attempt, dependencies=(dependency,))) + changed = replace(changed, dependencies=(dependency,)) + changed = replace( + changed, + receipts=(receipt, *changed.receipts[1:]), + attempts=tuple( + receipt.attempt if a.attempt_id == receipt.attempt.attempt_id else a + for a in changed.attempts + ), + ) + third = Search(control, "campaign", "changed-proof", snapshot, settings, adapter) + third.start(changed, seed=2) + changed_root = tree.candidate(changed, changed.root_id) + assert changed_root.artifact_digest == root.candidates[0].artifact_digest + assert third.read()[1]["initial_conditions"] != first.read()[1]["initial_conditions"] + with pytest.raises(ValueError, match="identical"): + examples((first.read()[1], third.finish("test"))) def test_snapshot_change_on_resume_is_rejected(runtime): diff --git a/tests/test_verification_plans.py b/tests/test_verification_plans.py index d87028f4..3ca3a6d3 100644 --- a/tests/test_verification_plans.py +++ b/tests/test_verification_plans.py @@ -44,6 +44,20 @@ def test_cross_branch_completion_counts_shared_preparation_once(): assert all(len(plan.job_ids) == 5 for plan in pairs) +def test_one_seed_can_complete_another_candidate_through_its_prerequisites(): + state, jobs = fixture() + jobs = tuple( + replace(job, prerequisites=("b-objective",)) if job.job_id == "a-quality" else job + for job in jobs + ) + plans = menu(state, jobs, max_targets=1) + coupled = next(plan for plan in plans if "a-objective" in plan.job_ids) + assert coupled.candidate_ids == ("a", "b") + assert coupled.job_ids.index("b-objective") < coupled.job_ids.index("a-quality") + assert coupled.cost == 15 and coupled.job_ids.count("setup") == 1 + assert len(coupled.job_ids) == 5 + + def exhaustive_oracle(jobs, budget, job_limit, target_limit): # Independent oracle enumerates all job subsets, not candidate-completion unions. result = {} diff --git a/tests/test_work_control.py b/tests/test_work_control.py index 4e768423..f5d6f74e 100644 --- a/tests/test_work_control.py +++ b/tests/test_work_control.py @@ -58,6 +58,39 @@ def test_unknown_cleanup_or_usage_retains_reservation_and_never_relaunches(contr assert reopened.read("campaign")["operations"]["work"]["status"] == "unknown" +def test_known_overrun_with_unknown_cleanup_blocks_other_domains_and_reconciles_once(control): + claim(control) + control.launch("campaign", "work") + control.settle( + "campaign", "work", {"result": "pending cleanup"}, actual_ms=950, cleanup_verified=False + ) + reopened = WorkControl(RunStore(control.store.root, control.store.namespace)) + result = reopened.read("campaign") + assert result["blocked"] + assert result["operations"]["work"]["status"] == "unknown" + assert reopened.available(result) == -50 + with pytest.raises(ValueError, match="blocked"): + claim(reopened, "other-domain", cap_ms=700, domains=("other-device",)) + reopened.create("other", {}, {}, cap_ms=1000, final_reserve_ms=100, deadline=time.time() + 60) + with pytest.raises(ValueError, match="leased"): + reopened.claim( + "other", 0, "work", category="verify", cap_ms=200, domains=("device",), binding={} + ) + # A later unknown observation cannot erase the retained lower bound. + reopened.settle("campaign", "work", {}, actual_ms=None, cleanup_verified=False) + assert reopened.available(reopened.read("campaign")) == -50 + with pytest.raises(ValueError, match="known usage"): + reopened.settle("campaign", "work", {}, actual_ms=50, cleanup_verified=True) + for _ in range(2): + reopened.settle("campaign", "work", {"done": True}, actual_ms=950, cleanup_verified=True) + result = reopened.read("campaign") + assert result["spent_ms"] == 950 and result["blocked"] + assert reopened.available(result) == -50 + reopened.claim( + "other", 0, "work", category="verify", cap_ms=200, domains=("device",), binding={} + ) + + def test_receipt_settlement_and_incumbent_are_atomic_and_idempotent(control): claim(control) control.launch("campaign", "work") From b84989494dfa69628aa918ed85653afc936a5493 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Thu, 19 Nov 2026 00:52:06 +0800 Subject: [PATCH 8/8] search: preserve terminal decisions and timeout accounting Retain protocol timeout receipts with supervisor cleanup facts so known usage settles and clean domains are released without relaxing overrun blocking. Resume committed ACCEPT decisions without sampling again and preserve consumed steps when restoring paid continuations. Reject development Skill evaluators before final claims. Cover process timeouts, uncertain cleanup, restart windows and final scope validation, and document the differences from the SpecRSI reference implementation. Agent-assisted-by: Codex (implementation, tests and reference analysis) Signed-off-by: amy-why-3459 --- docs/speculative-search.md | 57 +++++++++++ src/inferrsi/apps/search_benchmarks.py | 115 ++++++++++++---------- src/inferrsi/harness/search_campaign.py | 3 + src/inferrsi/harness/task_protocol.py | 12 ++- src/inferrsi/harness/vats.py | 15 ++- tests/test_search_benchmarks.py | 124 ++++++++++++++++++++++++ tests/test_task_protocol_child.py | 14 ++- tests/test_vats_runtime.py | 81 ++++++++++++++++ 8 files changed, 365 insertions(+), 56 deletions(-) diff --git a/docs/speculative-search.md b/docs/speculative-search.md index bf7020c8..5a81e14a 100644 --- a/docs/speculative-search.md +++ b/docs/speculative-search.md @@ -86,6 +86,9 @@ parent artifact, without waiting for parent quality. Each check binds candidate, contract, attempt, execution, reservation and domain. Settlement commits elapsed time, cleanup facts and evidence/incumbent changes in one transaction. Recovery continues the retained block cursor before making another Policy decision. +An ACCEPT already retained in the trace is terminal even if the controller +stopped before writing the completed status. Recovery finishes that decision +without sampling or charging another preparation, including forced ACCEPT. ACCEPT and limits return only complete admissible versions. Unstarted work stops when its conservative cap no longer fits. Unknown launched operations block @@ -126,6 +129,9 @@ suffix is Scorer-only. `paired_targets` requires the same source state, settings snapshot, adapter and continuation seed with different first actions. Both absolute suffix targets and observed relative differences train the Scorer. Caller-authored unreceipted pair targets are rejected. +The saved source step counts toward the continuation's original step limit and +random-stream position; omitting the prefix from its training trace does not +grant additional decision slots. These are real updates to a constrained CPU operator model. They are not LLM finetuning, prompt summarization or evidence of pretrained-model improvement. @@ -148,6 +154,10 @@ blocks further claims immediately, even with unknown cleanup; the retained hold is at least the largest known usage. Later reconciliation releases the domain and charges the final usage once, never below that known lower bound. Absolute deadlines include queueing and cleanup. +Engine protocol timeouts retain an INCONCLUSIVE adapter receipt with measured +elapsed time. Confirmed cleanup of both task and protocol permits settlement +and domain release; uncertain protocol cleanup keeps the reservation. A known +overrun still blocks the campaign, even after its domain has been released. In-process bounded planning, editing and training are checked after return; they are not forcibly preempted like subprocesses. Preflight freezing and initial setup are outside this campaign meter, a remaining lifecycle-cost limitation. @@ -171,6 +181,9 @@ original baseline. For Skill, supply a **holdout-bound adapter** and `Comparator.holdout_scope` (derived from the frozen data digest), never the development adapter or a replacement label. Exact bounded radix utility keeps quality before token use; reduced tokens cannot offset a quality regression. +The Skill evaluator validates its holdout binding before the scope is claimed +or data is accessed, rejecting development evaluators and missing/mismatched +holdout identities. After interruption, final consumes existing receipts only. Even an unstarted remaining check cannot be newly dispatched by recovery. Existing final results @@ -217,6 +230,50 @@ executes the existing comparator under the shared process executor and revalidat its independent retained report. Construction does not load holdout bytes. Provider readiness/credentials/authorization remain existing explicit requirements. +## Relation to the paper and reference implementation + +This is a constrained adaptation of the two-loop method, not a reproduction of +the reference model or experiments. The comparison below uses +[SpecRSI v2, section 3 and Algorithms S1-S3](https://arxiv.org/html/2610.16491v2#S3) +and the authors' [reference source](https://github.com/SpecRSI-GPR/SpecRSI/tree/ff0f0bb607e53ba76eebb9960df3d6942c33027e). +That source revision is research provenance, not an InferRSI campaign pin. + +| Mechanism | This implementation | Alignment and limits | +| --- | --- | --- | +| Frozen search and acceptance | Immutable descendants, separate real evidence, DRAFT/VERIFY/ACCEPT, complete-root fallback | Follows the central control contract; intermediate parents need not pass | +| Natural group learning | Independent matched searches, leave-one-out advantage, recorded actor probabilities, clipped objective and fixed-reference KL | Same objective structure; updates a small CPU operator model | +| Value supervision | Actual paid-state suffix returns and optional paid pair differences; forced trajectories excluded from actors | Same separation of targets; continuation recovery must preserve the original remaining work limits | +| Drafter | Softmax over a caller-supplied exact-replacement vocabulary | The reference uses generated edit tokens with a trainable LoRA adapter on a frozen language model; this vocabulary cannot propose a new edit outside its supplied operators | +| Policy and Scorer | Separate linear weights over eight aggregate features | The reference uses separate relation/message-passing and attention branches with semantic and observed-evidence inputs; these linear features omit those inputs | +| Solver and refinement | Exhaustive unions of completion seeds, one job per candidate/check; optional pairwise seed-union scoring | The reference supports MILP covers, singleton check closures, shared setup and iterative job-closure extensions; this restricted planner is not a general substitute | +| Training recipe | Caller-provided initial checkpoint; direct CPU gradient updates | No supervised warm-up pipeline or language-model training adapter; the reference's warm-up, role architecture and optimizer configuration are not reproduced | +| Resources and evaluation | One worker, millisecond reservations, whole Engine/Skill checks | No equivalent campaign-wide token/currency cap, arbitrary setup execution, parallel-worker study or matched reference benchmark runs | + +The architecture and training distinction can be inspected in the reference +[role model](https://github.com/SpecRSI-GPR/SpecRSI/blob/ff0f0bb607e53ba76eebb9960df3d6942c33027e/src/specrsi/learning/hf.py), +[relation heads](https://github.com/SpecRSI-GPR/SpecRSI/blob/ff0f0bb607e53ba76eebb9960df3d6942c33027e/src/specrsi/learning/heads.py), +[completion solver](https://github.com/SpecRSI-GPR/SpecRSI/blob/ff0f0bb607e53ba76eebb9960df3d6942c33027e/src/specrsi/planning/completion.py), +[menu construction](https://github.com/SpecRSI-GPR/SpecRSI/blob/ff0f0bb607e53ba76eebb9960df3d6942c33027e/src/specrsi/core/vats.py) +and [training recipe](https://github.com/SpecRSI-GPR/SpecRSI/blob/ff0f0bb607e53ba76eebb9960df3d6942c33027e/docs/REPRODUCIBILITY.md#training-the-reference-roles). +These were inspected as source, not executed as a reproduction experiment. + +Two limits matter when interpreting learning results. First, at the same state, +verification bundles with equal total cap and job count have identical features +in `vats.features`, hence identical linear Scorer values and Policy preferences. +Their candidate identity, observed failures and check interactions cannot affect +that comparison. Parameter updates alone therefore do not demonstrate learning +the paper's complementary-check value function. Second, `Settings.refine_queries` +defaults to zero, and the Engine CLI exposes no override. Its search does not use +the trained Scorer to refine the menu; the supplied Engine/Skill evaluators also +expose one indivisible check per candidate. Python callers can enable the bounded +refinement, but that does not add the missing semantic or relational inputs. + +Moving toward a reference reproduction requires explicit role/feature and menu +contracts, a qualified generative/trainable adapter with warm-up, lifecycle cost +accounting, and matched frozen-versus-trained search experiments on real tasks. +The existing recovery, evidence and holdout controls remain useful foundations; +passing them does not establish search efficacy or reproduce paper results. + ## Validation and issue completion ```bash diff --git a/src/inferrsi/apps/search_benchmarks.py b/src/inferrsi/apps/search_benchmarks.py index b89442d1..4af9580f 100644 --- a/src/inferrsi/apps/search_benchmarks.py +++ b/src/inferrsi/apps/search_benchmarks.py @@ -37,71 +37,84 @@ def __init__(self, package, frozen, *, arguments, python=sys.executable, prefix= } ) - def evaluate(self, candidate, check, output, deadline): - if check != "evaluation": - raise ValueError("Packaged engine protocol requires one full evaluation job") + def _check_resources(self, *, during=False): actual = { p.relative_to(self.frozen).as_posix(): sha256(p) for p in self.frozen.rglob("*") if p.is_file() } if actual != self.plan["frozen_files"]: - raise ValueError("Frozen benchmark resources changed") + suffix = " during execution" if during else "" + raise ValueError("Frozen benchmark resources changed" + suffix) + + def evaluate(self, candidate, check, output, deadline): + if check != "evaluation": + raise ValueError("Packaged engine protocol requires one full evaluation job") + self._check_resources() output.mkdir(parents=True) candidates.retain(candidate / "candidate.py", output / "candidate.py") - args = task_protocol.run( - self.frozen, - self.plan["frozen_files"]["optimization.py"], - "arguments", - deadline=time.monotonic() + max(0, deadline - time.time()), - candidate=output / "candidate.py", - output=output / "measurement", - settings=self.plan["evaluation_settings"], - ) from inferrsi.harness import optimization_session - command = ( - *self.prefix, - self.python, - str(Path(optimization_session.__file__).resolve()), - str(self.frozen / "benchmark.py"), - *args, - ) - environment = runtime_environment() - environment["PYTHONDONTWRITEBYTECODE"] = "1" - execution = execute_trial( - command, - output, - output / "execution", - time.monotonic() + max(0, deadline - time.time()), - None, - environment=environment, - include_cleanup=True, - ) - if { - p.relative_to(self.frozen).as_posix(): sha256(p) - for p in self.frozen.rglob("*") - if p.is_file() - } != self.plan["frozen_files"]: - raise ValueError("Frozen benchmark resources changed during execution") - assessed = task_protocol.run( - self.frozen, - self.plan["frozen_files"]["optimization.py"], - "assess", - deadline=time.monotonic() + max(0, deadline - time.time()), - directory=output, - settings=self.plan["evaluation_settings"], - execution=execution, - ) - if execution["cleanup_verified"] is not True: + execution = None + cleanup = True + try: + args = task_protocol.run( + self.frozen, + self.plan["frozen_files"]["optimization.py"], + "arguments", + deadline=time.monotonic() + max(0, deadline - time.time()), + candidate=output / "candidate.py", + output=output / "measurement", + settings=self.plan["evaluation_settings"], + ) + command = ( + *self.prefix, + self.python, + str(Path(optimization_session.__file__).resolve()), + str(self.frozen / "benchmark.py"), + *args, + ) + environment = runtime_environment() + environment["PYTHONDONTWRITEBYTECODE"] = "1" + execution = execute_trial( + command, + output, + output / "execution", + time.monotonic() + max(0, deadline - time.time()), + None, + environment=environment, + include_cleanup=True, + ) + cleanup = execution["cleanup_verified"] + self._check_resources(during=True) + assessed = task_protocol.run( + self.frozen, + self.plan["frozen_files"]["optimization.py"], + "assess", + deadline=time.monotonic() + max(0, deadline - time.time()), + directory=output, + settings=self.plan["evaluation_settings"], + execution=execution, + ) + except task_protocol.TaskProtocolError as error: + if error.kind != "timeout": + raise + # Retain a real timeout as INCONCLUSIVE even when no time remains + # for assessment. Cleanup covers both the task and protocol process; + # an unknown protocol exit cannot release the measurement domain. + cleanup = cleanup is True and error.cleanup_verified is True + assessed = {"verdict": "INCONCLUSIVE", "score": None, "reason": "protocol_timeout"} + self._check_resources(during=True) + if cleanup is not True: assessed = {"verdict": "INCONCLUSIVE", "score": None} atomic_write_json(output / "assessment.json", assessed) return { "verdict": assessed["verdict"], "objective": assessed.get("score"), - "cleanup_verified": execution["cleanup_verified"], + "cleanup_verified": cleanup, "assessment_sha256": sha256(output / "assessment.json"), - "execution_sha256": sha256(output / "execution/execution.json"), + "execution_sha256": sha256(output / "execution/execution.json") + if execution is not None else None, } @@ -142,6 +155,10 @@ def __init__(self, comparator, *, split, policy, token_cap, task_count): } ) + def validate_final(self, scope): + if self.split != "holdout" or not self.holdout_scope or scope != self.holdout_scope: + raise ValueError("Skill final requires a holdout-bound evaluator and its frozen scope") + def evaluate(self, candidate, check, output, deadline): if check != "evaluation": raise ValueError("Skill comparison is an indivisible evaluation job") diff --git a/src/inferrsi/harness/search_campaign.py b/src/inferrsi/harness/search_campaign.py index 96897c51..bf9c205d 100644 --- a/src/inferrsi/harness/search_campaign.py +++ b/src/inferrsi/harness/search_campaign.py @@ -252,6 +252,9 @@ def finalize(control, campaign, adapter, selected, *, holdout_scope, accept_resu doc = control.read(campaign) if doc["final_result"] is not None: return doc["final_result"] + validate_final = getattr(adapter.evaluator, "validate_final", None) + if validate_final is not None: + validate_final(holdout_scope) required_scope = getattr(adapter.evaluator, "holdout_scope", None) if required_scope is not None and holdout_scope != required_scope: raise ValueError("Final scope must match the evaluator's frozen holdout data identity") diff --git a/src/inferrsi/harness/task_protocol.py b/src/inferrsi/harness/task_protocol.py index 792b5564..df2684a9 100644 --- a/src/inferrsi/harness/task_protocol.py +++ b/src/inferrsi/harness/task_protocol.py @@ -42,10 +42,14 @@ class TaskProtocolError(ValueError): starts with the kind, so transport, identity and result failures stay distinct. """ - def __init__(self, kind: str, message: str, *, error_type: str | None = None): + def __init__(self, kind: str, message: str, *, error_type: str | None = None, + cleanup_verified: bool | None = None): super().__init__(message if kind == "protocol_exception" else f"{kind}: {message}") self.kind = kind self.type_name = error_type if kind == "protocol_exception" else type(self).__name__ + # Only the process supervisor (or conclusive refusal to launch) can + # establish this fact. Unknown cleanup must retain the caller's lease. + self.cleanup_verified = cleanup_verified def _tail(path: Path) -> str: @@ -102,7 +106,8 @@ def run(root: Path, protocol_sha256: str, operation: str, *, deadline=None, **ar raise TaskProtocolError("request", "deadline must be finite monotonic time") timeout = min(timeout, deadline - time.monotonic()) if timeout <= 0: - raise TaskProtocolError("timeout", "deadline expired before protocol execution") + raise TaskProtocolError("timeout", "deadline expired before protocol execution", + cleanup_verified=True) if sha256(path) != protocol_sha256: raise TaskProtocolError("identity", "protocol bytes differ from the frozen digest") request = {"schema_version": protocol_child.SCHEMA_VERSION, "operation": operation, @@ -135,7 +140,8 @@ def run(root: Path, protocol_sha256: str, operation: str, *, deadline=None, **ar raise KeyboardInterrupt if record["status"] == "timeout": raise TaskProtocolError( - "timeout", f"{operation} exceeded {timeout:g} s; {stderr}") + "timeout", f"{operation} exceeded {timeout:g} s; {stderr}", + cleanup_verified=record["domain_empty"] if record["pid"] is not None else True) if record["status"] not in ("completed", "failed"): raise TaskProtocolError("launch", f"{record['status']}: {record['error']}") if record["returncode"] != 0: diff --git a/src/inferrsi/harness/vats.py b/src/inferrsi/harness/vats.py index ab8b67ac..e201860c 100644 --- a/src/inferrsi/harness/vats.py +++ b/src/inferrsi/harness/vats.py @@ -78,6 +78,11 @@ def depth(state, node): return result +def decision_count(record): + """Include the natural prefix omitted from a forced continuation's trace.""" + return record.get("continuation", {}).get("step", 0) + len(record["trace"]) + + def features(state, action, available, settings): """Fixed public features; ACCEPT has a fixed zero Policy logit.""" if action.kind == "accept": @@ -475,7 +480,7 @@ def pending(state): # the only recovery route, including when writing the state below fails. if kind == "draft": rng = random.Random( - f"{search['seed']}:{len(current.candidates)}:{len(search['trace'])}" + f"{search['seed']}:{len(current.candidates)}:{decision_count(search)}" ) result = self.adapter.draft( node, current, self.snapshot, rng, operation, claim["deadline"] @@ -510,6 +515,10 @@ def run(self, *, forced_action=None): for op in doc["operations"].values() ): raise ValueError("Unresolved claim: recover without relaunch") + if search["trace"] and search["trace"][-1]["action"]["kind"] == "accept": + # Preparation/copy already committed this terminal decision. + # A crash before finish must not sample or pay for another action. + return self.finish("forced_accept" if search["trace"][-1]["forced"] else "accept") if search["active_action"] is not None: self._continue_action() continue @@ -520,7 +529,7 @@ def run(self, *, forced_action=None): current.contract.max_attempts - len(current.attempts) - search["draft_attempts"] ) if ( - len(search["trace"]) >= self.settings.max_steps + decision_count(search) >= self.settings.max_steps or attempts_left <= 0 or remaining <= self.settings.preparation_ms or time.time() >= doc["deadline"] @@ -558,7 +567,7 @@ def run(self, *, forced_action=None): vectors = tuple( features(current, action, remaining, self.settings) for action in actions ) - rng = random.Random(f"{search['seed']}:policy:{len(search['trace'])}") + rng = random.Random(f"{search['seed']}:policy:{decision_count(search)}") choice = self.snapshot.choose( "policy", [action.key for action in actions], diff --git a/tests/test_search_benchmarks.py b/tests/test_search_benchmarks.py index bf957bb1..515b35ea 100644 --- a/tests/test_search_benchmarks.py +++ b/tests/test_search_benchmarks.py @@ -1,4 +1,5 @@ import json +import sys import time from types import SimpleNamespace @@ -8,6 +9,13 @@ from inferrsi.apps.search_benchmarks import EngineEvaluator, SkillEvaluator from inferrsi.apps.speculative_search import command from inferrsi.benchmark.improvement import SelectionPolicy +from inferrsi.harness.run_store import RunStore +from inferrsi.harness.scenarios.external import execute_trial +from inferrsi.harness.search_adapter import ArtifactAdapter, Edit +from inferrsi.harness.search_campaign import finalize, qualify_root +from inferrsi.harness.task_protocol import TaskProtocolError +from inferrsi.harness.work_control import WorkControl +from tests.fixtures.speculative_search import contract def skill_report(base_tokens=100, tokens=80, *, verdict="PASS", outcome="unchanged"): @@ -132,6 +140,122 @@ def execute(command, cwd, output, deadline, *args, **kwargs): ("assess", 1110.0, 5.0)] +def test_engine_known_timeout_settles_usage_and_releases_domain(tmp_path, monkeypatch): + evaluator = EngineEvaluator( + "qwen21-denoise-npu", tmp_path / "frozen", + arguments=["--pairs", "1", "--samples", "1", "--warmups", "0"], + ) + executions = [] + + def timeout(command, cwd, output, deadline, *args, **kwargs): + # Exercise real process cleanup without model/device execution. + result = execute_trial((sys.executable, "-c", "import time; time.sleep(60)"), + cwd, output, deadline, *args, **kwargs) + executions.append(result) + return result + + monkeypatch.setattr("inferrsi.apps.search_benchmarks.execute_trial", timeout) + source = tmp_path / "source" + source.mkdir() + (source / "candidate.py").write_text("value = 1\n") + adapter = ArtifactAdapter(tmp_path / "artifacts", [Edit("edit", "candidate.py", "1", "2")], + {"evaluation": 2000}, evaluator, domain="cpu-timeout-test") + root = adapter.import_root(source, snapshot="test-snapshot", context={}) + control = WorkControl(RunStore(tmp_path / "store", "timeout-test")) + control.create("campaign", {}, {"searches": {}}, cap_ms=20000, final_reserve_ms=3000, + deadline=time.time() + 20) + configuration = contract(checks=("evaluation",), objective_check="evaluation") + with pytest.raises(ValueError, match="Root lacks complete"): + qualify_root(control, "campaign", adapter, root, configuration) + assert executions[0]["status"] == "timeout" and executions[0]["cleanup_verified"] is True + reopened = WorkControl(RunStore(control.store.root, control.store.namespace)) + doc = reopened.read("campaign") + operation, settled = next(iter(doc["operations"].items())) + assert settled["status"] == "settled" and settled["actual_ms"] > 0 + assert doc["spent_ms"] == settled["actual_ms"] + assert settled["receipt"]["verdict"] == "INCONCLUSIVE" + assert settled["receipt"]["cleanup_verified"] is True + assert adapter.recover(operation, {"kind": "verify", "binding": settled["binding"]}) == ( + settled["receipt"]) + with reopened.store._connection() as db: + assert db.execute("SELECT 1 FROM work_domains").fetchone() is None + # Deadline overruns remain blocking; a clean timeout is not a free retry. + assert doc["blocked"] + + +@pytest.mark.parametrize("stage", ["arguments", "assess"]) +@pytest.mark.parametrize("protocol_cleanup", [True, False, None]) +def test_engine_protocol_timeout_preserves_cleanup_facts( + tmp_path, monkeypatch, stage, protocol_cleanup +): + evaluator = EngineEvaluator( + "qwen21-denoise-npu", tmp_path / "frozen", + arguments=["--pairs", "1", "--samples", "1", "--warmups", "0"], + ) + source = tmp_path / "source" + source.mkdir() + (source / "candidate.py").write_text("candidate") + + def protocol(root, digest, operation, **kwargs): + if operation == stage: + raise TaskProtocolError( + "timeout", "injected timeout", cleanup_verified=protocol_cleanup) + return [] + + def execute(command, cwd, output, *args, **kwargs): + output.mkdir(parents=True) + (output / "execution.json").write_text("{}") + return {"status": "completed", "returncode": 0, "cleanup_verified": True} + + monkeypatch.setattr("inferrsi.apps.search_benchmarks.task_protocol.run", protocol) + monkeypatch.setattr("inferrsi.apps.search_benchmarks.execute_trial", execute) + result = evaluator.evaluate(source, "evaluation", tmp_path / "output", time.time() + 10) + assert result["verdict"] == "INCONCLUSIVE" and result["objective"] is None + assert result["cleanup_verified"] is (protocol_cleanup is True) + assert (result["execution_sha256"] is None) == (stage == "arguments") + + +@pytest.mark.parametrize("split, declared_scope, supplied_scope, valid", [ + ("development", "skill-data:holdout", "skill-data:holdout", False), + ("holdout", None, "skill-data:holdout", False), + ("holdout", "skill-data:holdout", "replacement-label", False), + ("holdout", "skill-data:holdout", "skill-data:holdout", True), +]) +def test_skill_final_requires_holdout_before_claim_or_access( + tmp_path, split, declared_scope, supplied_scope, valid +): + accesses = [] + + def compare(*args, split, **kwargs): + accesses.append(split) + return skill_report(tokens=100), True + + comparator = SimpleNamespace(identity="test-comparator", run=compare, + holdout_scope=declared_scope) + evaluator = SkillEvaluator(comparator, split=split, policy=SelectionPolicy(1, 0), + token_cap=1000, task_count=1) + adapter = ArtifactAdapter(tmp_path / "artifacts", [Edit("edit", "skill.txt", "old", "new")], + {"evaluation": 3000}, evaluator, domain="skill-test") + source = tmp_path / "source" + source.mkdir() + (source / "skill.txt").write_text("old") + root = adapter.import_root(source, snapshot="test-snapshot", context={}) + control = WorkControl(RunStore(tmp_path / "store", "skill-test")) + before = control.create("campaign", {}, {}, cap_ms=10000, final_reserve_ms=3000, + deadline=time.time() + 10) + kwargs = dict(holdout_scope=supplied_scope, accept_result=lambda results: True) + if valid: + result = finalize(control, "campaign", adapter, root, **kwargs) + assert result["verdict"] == "PASS" and result["promoted"] + assert accesses == ["holdout"] + assert finalize(control, "campaign", adapter, root, **kwargs) == result + assert accesses == ["holdout"] + else: + with pytest.raises(ValueError, match="holdout-bound"): + finalize(control, "campaign", adapter, root, **kwargs) + assert control.read("campaign") == before and not accesses + + def test_skill_comparator_preparation_does_not_access_holdout(tmp_path): from tests.fixtures.harness_improvement_demo import configuration from tools.speculative_skill import Comparator diff --git a/tests/test_task_protocol_child.py b/tests/test_task_protocol_child.py index c7689803..cf417151 100644 --- a/tests/test_task_protocol_child.py +++ b/tests/test_task_protocol_child.py @@ -225,6 +225,7 @@ def test_campaign_deadline_bounds_the_protocol_timeout(tmp_path): started = time.monotonic() error = failure(tmp_path, digest, deadline=started + 1.0, arguments=[]) assert error.kind == "timeout" + assert error.cleanup_verified is True assert time.monotonic() - started < 10 @@ -232,7 +233,18 @@ def test_expired_campaign_deadline_never_starts_protocol(tmp_path, monkeypatch): digest = write_protocol(tmp_path, "def validate_arguments(a):\n return {}\n") monkeypatch.setattr(task_protocol.local_process, "_supervise", lambda *a, **k: pytest.fail("expired protocol must not start")) - assert failure(tmp_path, digest, deadline=0, arguments=[]).kind == "timeout" + error = failure(tmp_path, digest, deadline=0, arguments=[]) + assert error.kind == "timeout" and error.cleanup_verified is True + + +@pytest.mark.parametrize("cleanup", [False, None]) +def test_protocol_timeout_retains_uncertain_cleanup(tmp_path, monkeypatch, cleanup): + digest = write_protocol(tmp_path, "def validate_arguments(a):\n return {}\n") + monkeypatch.setattr(task_protocol.local_process, "_supervise", lambda *a, **k: { + "status": "timeout", "pid": 123, "domain_empty": cleanup, + }) + error = failure(tmp_path, digest) + assert error.kind == "timeout" and error.cleanup_verified is cleanup @pytest.mark.parametrize("deadline", [float("nan"), float("inf"), -float("inf"), True, "soon"]) diff --git a/tests/test_vats_runtime.py b/tests/test_vats_runtime.py index 522667cf..fc3cd417 100644 --- a/tests/test_vats_runtime.py +++ b/tests/test_vats_runtime.py @@ -150,6 +150,36 @@ def crash(campaign, operation, receipt, **kwargs): assert len(list((adapter.root / "candidates").iterdir())) == before +def test_durable_accept_recovers_without_another_policy_decision(runtime, monkeypatch): + control, adapter, snapshot, settings, root = runtime + snapshot = replace(snapshot, policy=(0.0,) * 8) + root = replace(root, contract=replace( + root.contract, improver_snapshot=digest(snapshot.document()))) + search = Search(control, "campaign", "search", snapshot, settings, adapter) + # This seed selects ACCEPT first, but would select DRAFT on a second decision. + search.start(root, seed=7, collect=True) + + def crash(reason): + assert reason == "accept" + raise RuntimeError("crash after durable ACCEPT") + + monkeypatch.setattr(search, "finish", crash) + with pytest.raises(RuntimeError, match="durable ACCEPT"): + search.run() + before = control.read("campaign") + reopened = WorkControl(RunStore(control.store.root, control.store.namespace)) + resumed = Search(reopened, "campaign", "search", snapshot, settings, adapter) + record = resumed.run() + assert record["status"] == "complete" and record["stop_reason"] == "accept" + assert record["selected"] == root.root_id + assert record["trace"] == before["state"]["searches"]["search"]["trace"] + assert reopened.read("campaign")["operations"] == before["operations"] + assert reopened.read("campaign")["spent_ms"] == before["spent_ms"] + after = reopened.read("campaign") + assert resumed.run() == record + assert reopened.read("campaign") == after + + @pytest.mark.parametrize("kind", ["draft", "verify"]) def test_expired_launch_recovers_unstarted_block_and_finishes_with_root(runtime, monkeypatch, kind): control, adapter, snapshot, settings, root = runtime @@ -364,6 +394,57 @@ def test_paid_continuations_are_metered_matched_and_excluded_from_actors(runtime assert trained.scorer != snapshot.scorer +def test_late_continuation_preserves_the_consumed_step_budget(runtime): + from inferrsi.harness.gvo import suffix_examples + from inferrsi.harness.search_continuations import continue_from + from inferrsi.harness.vats import Action + + control, adapter, snapshot, settings, root = runtime + settings = replace(settings, max_steps=2) + source = Search(control, "campaign", "search", snapshot, settings, adapter) + source.start(root, seed=12, collect=False) + original = source.run() + assert [s["action"]["kind"] for s in original["trace"]] == ["draft", "verify"] + action = next(Action(**raw) for raw in original["trace"][1]["menu"] + if raw["kind"] == "draft") + args = (source, 1, action.key, "late-suffix") + record = continue_from(*args, copy_cap_ms=3000, seed=91) + assert record["continuation"]["step"] == 1 + assert [s["action"]["kind"] for s in record["trace"]] == ["draft"] + assert record["stop_reason"] == "limit" and record["selected"] == root.root_id + assert record["gain"] == record["initial_gain"] == 0 + assert suffix_examples(record, expense_weight=0, elapsed_weight=0)[0][1] == 0 + before = control.read("campaign") + assert continue_from(*args, copy_cap_ms=3000, seed=91) == record + assert control.read("campaign") == before + + +def test_forced_accept_recovers_after_copy_without_dispatch(runtime, monkeypatch): + from inferrsi.harness.search_continuations import continue_from + from inferrsi.harness.vats import Action + + control, adapter, snapshot, settings, root = runtime + source = Search(control, "campaign", "search", snapshot, settings, adapter) + source.start(root, seed=12, collect=False) + original = source.run() + action = next(Action(**raw) for raw in original["trace"][0]["menu"] + if raw["kind"] == "accept") + args = (source, 0, action.key, "accept-suffix") + with monkeypatch.context() as patch: + def crash(*args): + raise RuntimeError("crash after continuation copy") + + patch.setattr(Search, "finish", crash) + with pytest.raises(RuntimeError, match="continuation copy"): + continue_from(*args, copy_cap_ms=3000, seed=91) + before = control.read("campaign") + record = continue_from(*args, copy_cap_ms=3000, seed=91) + assert record["stop_reason"] == "forced_accept" and record["selected"] == root.root_id + assert len(record["trace"]) == 1 + assert control.read("campaign")["operations"] == before["operations"] + assert control.read("campaign")["spent_ms"] == before["spent_ms"] + + def test_root_invalidation_is_persisted_and_cannot_return_an_invalid_root(runtime): control, adapter, snapshot, settings, root = runtime dependency = tree.Dependency("environment", 0)