diff --git a/agentdescent/__init__.py b/agentdescent/__init__.py index fdb67ca2..95ce3a2c 100644 --- a/agentdescent/__init__.py +++ b/agentdescent/__init__.py @@ -31,6 +31,7 @@ AuditedReward, AuditRecord, AuditStore, DeferredOracle, GoldAnswer, NullOracle, Purpose, RenderTap, resolve_from_mapping, verifier_fingerprint, residual_bias, + BoostState, VerifierBoost, isotonic_fit, ) from . import backends, dataloader # submodules: agentdescent.dataloader.hf_rows(...) from .dataloader import Dataset, split_dataset diff --git a/agentdescent/audit/__init__.py b/agentdescent/audit/__init__.py index 7803de09..6e3ad252 100644 --- a/agentdescent/audit/__init__.py +++ b/agentdescent/audit/__init__.py @@ -67,6 +67,7 @@ """ from .wiring import Audit, attach +from .boost import BoostState, VerifierBoost, isotonic_fit from .calibrator import (STALE_INFLATION, Calibrator, Rectification, population_resid_sd) from .coverage import (CALIBRATION_CEILING, CALIBRATION_FLOOR, MIN_UNSEEN, diff --git a/agentdescent/audit/boost.py b/agentdescent/audit/boost.py new file mode 100644 index 00000000..c7642cab --- /dev/null +++ b/agentdescent/audit/boost.py @@ -0,0 +1,351 @@ +"""The verifier bootstrap flywheel: the audit's labels improve the verifier. + +The audit package already turns labelled records into a *correction of the +loop*: `Calibrator` produces a rectification (``delta_hat``, ``se``, +``resid_sd``) that `RectifiedAcceptance` uses to discount the gate's evidence. +What it never does is turn those labels into a *better verifier* -- the +correction is subtracted at the decision, and the scorer itself is left +exactly as it was. This module closes that gap under a bootstrap gate. + +A :class:`VerifierBoost` wraps any :class:`~agentdescent.policies. +VerifierProtocol` (the shipped :class:`~agentdescent.verifier.ThreeLayerVerifier`) +and, from the audit's **IMPROVEMENT** pool, fits a monotone calibration curve + + g: verifier_score -> oracle-consistent score + +(isotonic regression -- pool-adjacent-violators, numpy-only, so score order is +preserved). The curve is **activated only if it provably reduces residual +error on a held-out slice of the same pool**: ``E[(g(f) - Y)^2]`` must beat +``E[(f - Y)^2]`` by ``min_gain``. That is the bootstrap gate, and it is what +keeps a self-improving verifier honest -- a curve that learned to be +confident-but-wrong fails validation and is recorded as a rejection, not +installed. + +**What it may and may not touch** (the governance invariants): + +* It trains on the **IMPROVEMENT pool only**. The CALIBRATION pool is never + used for training: that pool exists to *measure* the verifier, and using its + labels to *change* the verifier would let a verifier fit its own judge + (``store.for_calibration`` already enforces the two-pool separation). +* It corrects the **cheap and learned layers** -- the scores the aggregator + uses to rank candidates (conflict resolution, the fusion tournament) and the + audit's own trust/priority signal. A calibrated cheap layer sends better + candidates to the commit gate without ever touching the gate. +* It delegates ``eval_counts`` and ``full_eval`` **raw**. The commit decision + stays on the raw verifier's full-set judgment, rectified by the audit. The + correction never substitutes for the oracle's authority over what commits. +* It declares itself as a **new verifier version** when it activates + (:meth:`version_tag`), so the audit's ``VerifierWatch`` withdraws the old + calibration and fresh CALIBRATION labels measure the boosted verifier. That + is the recursive part done honestly: the flywheel's output is measured by + the same audit that produced its training labels, and a boost that is not + actually better is caught twice -- by the bootstrap gate at install time and + by the calibration of the new version afterwards. + +The uncertainty the learned layer reports is raised to at least the boost's +own validation residual (``max(resid_sd, uncertainty)``) when active, so a +poorly-fitting boost never under-claims uncertainty to the audit's priority +formula (``blast_radius * uncertainty / trust``). A tight residual leaves a +larger inner uncertainty unchanged; a loose residual raises the floor. This is +a safety floor, not a claim that a good boost spends less oracle budget. + +**Scale assumption.** The curve is fit on per-unit ``AuditRecord.verifier_score`` +(the audited reward) and applied to artifact-level ``cheap_eval`` / +``learned_eval``. Callers must ensure those layer scores live on the same +scale as the audited per-unit reward; for a non-linear ``g``, +``g(mean f) != mean g(f)``. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import random +import time +from dataclasses import dataclass +from typing import Any, Callable, List, Optional, Sequence, Tuple + +import numpy as np + +from ..evolvable import Evolvable +from .store import AuditStore + +__all__ = ["BoostState", "VerifierBoost", "isotonic_fit"] + + +# -- the correction learner ----------------------------------------------------- + + +def isotonic_fit(x: Sequence[float], y: Sequence[float]) -> Tuple[np.ndarray, np.ndarray]: + """Monotone non-decreasing regression of ``y`` on ``x`` (PAV, numpy-only). + + Pool-adjacent-violators: sort by ``x``, then repeatedly merge adjacent + blocks whose fitted value has fallen below its predecessor's. Returns the + block boundary x-coordinates and the fitted step values -- a piecewise + constant, monotone curve that never inverts score order (flat PAV blocks may + still turn distinct raw scores into ties), which is exactly what a + *ranking* correction must do (an arbitrary regressor could invert two + candidates' order and break the tournament). + + Deliberately numpy rather than scipy: the audit package's PPI estimator is + the only numpy consumer today, and scipy is not a dependency. + """ + x = np.asarray(x, dtype=float) + y = np.asarray(y, dtype=float) + order = np.argsort(x, kind="mergesort") + x, y = x[order], y[order] + x_sum: List[float] = [] + w: List[float] = [] + y_sum: List[float] = [] + for xi, yi in zip(x, y): + x_sum.append(float(xi)) + w.append(1.0) + y_sum.append(float(yi)) + while len(w) >= 2 and y_sum[-1] / w[-1] < y_sum[-2] / w[-2]: + x_sum[-2] += x_sum[-1] + w[-2] += w[-1] + y_sum[-2] += y_sum[-1] + del x_sum[-1], w[-1], y_sum[-1] + xs = np.array([x_sum[k] / w[k] for k in range(len(w))]) + vals = np.array([y_sum[k] / w[k] for k in range(len(w))]) + return xs, vals + + +def _mse(a: Sequence[float], b: Sequence[float]) -> float: + if not a or not b or len(a) != len(b): + return math.inf + return float(np.mean((np.asarray(a, dtype=float) + - np.asarray(b, dtype=float)) ** 2)) + + +def _finite(v: Optional[float]) -> bool: + try: + return v is not None and math.isfinite(float(v)) + except (TypeError, ValueError): + return False + + +@dataclass(frozen=True) +class BoostState: + """One fit's verdict: what the bootstrap gate decided, and why.""" + + active: bool + fitted: bool + n_train: int + n_val: int + error_before: float + error_after: float + reason: str + fit_at: float + + +class VerifierBoost: + """A verifier that improves itself from audited labels, under a bootstrap gate. + + Wraps a :class:`~agentdescent.policies.VerifierProtocol`. On + :meth:`retrain` it reads the store's IMPROVEMENT pool, fits the monotone + calibration curve, validates it on a held-out slice, and either activates + it (the cheap/learned layers become ``g(score)``) or records the rejection. + ``eval_counts`` / ``full_eval`` are delegated raw -- the commit gate never + sees the correction. + + ``store`` is the same :class:`~agentdescent.audit.store.AuditStore` the + audit writes to, so the boost's training pool and the audit's measurement + pool are one store with two asserted halves. + """ + + def __init__( + self, + verifier: Any, + store: AuditStore, + *, + min_labels: int = 50, + min_val_labels: int = 10, + validation_frac: float = 0.2, + min_gain: float = 0.05, + seed: int = 0, + max_records: int = 2000, + auto: bool = True, + version_for_training: Optional[Callable[[], Optional[str]]] = None, + ) -> None: + if not (0.0 <= min_gain <= 1.0): + raise ValueError(f"min_gain must be in [0, 1], got {min_gain}") + if not (0.0 < validation_frac < 1.0): + raise ValueError(f"validation_frac must be in (0, 1), got {validation_frac}") + if int(min_labels) <= int(min_val_labels): + raise ValueError( + f"min_labels ({min_labels}) must be greater than " + f"min_val_labels ({min_val_labels}) so the training split " + f"cannot be emptied by the validation holdout") + self._inner = verifier + self.store = store + self.min_labels = int(min_labels) + self.min_val_labels = int(min_val_labels) + self.validation_frac = float(validation_frac) + self.min_gain = float(min_gain) + self.seed = int(seed) + self.max_records = int(max_records) + self._version_for_training = version_for_training + self._xs: Optional[np.ndarray] = None + self._vals: Optional[np.ndarray] = None + self._fit_hash: str = "" + self._resid_sd = math.inf + self._state = BoostState(False, False, 0, 0, math.inf, math.inf, + "not fitted yet", 0.0) + # Preserve the wrapped verifier's full-set equivalence declaration + # (missing / legacy False stay independent measurements -- #273 P1). + from ..verifier import shares_eval_counts + self.full_eval_matches_counts = shares_eval_counts(verifier) + if auto: + self.retrain() + + # -- the flywheel --------------------------------------------------------- + + def retrain(self, verifier_version: Optional[str] = None) -> BoostState: + """Refit the calibration curve from the current IMPROVEMENT pool. + + The bootstrap gate: fit on the train slice, validate on a held-out + slice, activate only if the corrected validation error beats the raw + by ``min_gain``. A failed gate deactivates the boost and keeps the + wrapped verifier raw -- a curve that does not help is recorded as a + rejection, not installed. + + ``verifier_version`` (or ``version_for_training``) filters the pool to + labels scored by the judge currently under measurement, so an older + verifier's labels cannot train the curve for a newer one. + """ + version = verifier_version + if version is None and self._version_for_training is not None: + version = self._version_for_training() + recs = [r for r in self.store.for_improvement(version) + if r.resolved and _finite(r.verifier_score) + and _finite(r.oracle_score)] + recs = recs[-self.max_records:] + if len(recs) < self.min_labels: + return self._deactivate( + f"insufficient improvement labels ({len(recs)} < " + f"{self.min_labels})") + rng = random.Random(self.seed) + shuffled = list(recs) + rng.shuffle(shuffled) + n_val = max(self.min_val_labels, int(len(shuffled) * self.validation_frac)) + # Keep at least one training point: n_val must not consume the pool. + n_val = min(n_val, len(shuffled) - 1) + train, val = shuffled[n_val:], shuffled[:n_val] + if not train: + return self._deactivate("empty training split after validation holdout") + try: + xs = np.array([r.verifier_score for r in train], dtype=float) + ys = np.array([r.oracle_score for r in train], dtype=float) + fit_xs, fit_vals = isotonic_fit(xs, ys) + except Exception as e: # noqa: BLE001 - a broken fit is a rejection, not a crash + return self._deactivate(f"isotonic fit failed: {type(e).__name__}: {e}") + if fit_xs.size == 0 or fit_vals.size == 0: + return self._deactivate("isotonic fit returned an empty curve") + + def _apply_candidate(f: float) -> float: + return float(np.interp(float(f), fit_xs, fit_vals, + left=float(fit_vals[0]), + right=float(fit_vals[-1]))) + + error_before = _mse([r.verifier_score for r in val], + [r.oracle_score for r in val]) + error_after = _mse([_apply_candidate(r.verifier_score) for r in val], + [r.oracle_score for r in val]) + if error_after <= error_before * (1.0 - self.min_gain): + digest = hashlib.sha256(json.dumps({ + "xs": fit_xs.tolist(), "vals": fit_vals.tolist(), + "n_train": len(train)}, sort_keys=True).encode()).hexdigest() + # Assign curve + state together only after the gate passes, so a + # concurrent cheap_eval never applies an unvalidated curve under an + # older active version_tag. + self._xs, self._vals = fit_xs, fit_vals + self._fit_hash = digest[:12] + self._resid_sd = math.sqrt(error_after) + self._state = BoostState(True, True, len(train), len(val), + error_before, error_after, + "validated", time.time()) + return self._state + return self._deactivate( + f"bootstrap gate: corrected error {error_after:.4f} is not below " + f"raw {error_before:.4f} by min_gain {self.min_gain}") + + def _deactivate(self, reason: str) -> BoostState: + self._xs = None + self._vals = None + self._fit_hash = "" + self._resid_sd = math.inf + self._state = BoostState(False, True, 0, 0, math.inf, math.inf, + reason, time.time()) + return self._state + + @property + def is_active(self) -> bool: + return self._state.active + + def _apply(self, f: float) -> float: + """Map a verifier score through the fitted calibration curve.""" + if self._xs is None or self._vals is None: + return float(f) + return float(np.interp(float(f), self._xs, self._vals, + left=float(self._vals[0]), + right=float(self._vals[-1]))) + + def version_tag(self) -> str: + """The audit's ``version_extra``: changes whenever the boost changes. + + Inactive and active fits produce different tags, and an active tag is + stable across calls until the next ``retrain`` -- so the audit's + fingerprint (and with it ``VerifierWatch``'s calibration withdrawal) + tracks exactly the state the loop is using.""" + if not self._state.active: + return "boost:inactive" + return f"boost:active:{self._fit_hash}" + + def status(self) -> dict: + s = self._state + return { + "active": s.active, + "fitted": s.fitted, + "n_train": s.n_train, + "n_val": s.n_val, + "error_before": s.error_before, + "error_after": s.error_after, + "resid_sd": self._resid_sd, + "reason": s.reason, + "fit_at": s.fit_at, + "version_tag": self.version_tag(), + } + + # -- the verifier surface -------------------------------------------------- + # + # The correction touches the layers the aggregator uses to *rank* and the + # audit uses for its trust signal; the commit gate (`eval_counts` / + # `full_eval`) is delegated raw. See the module docstring for why. + + def rule_eval(self, artifact: Evolvable) -> float: + return self._inner.rule_eval(artifact) + + def cheap_eval(self, artifact: Evolvable) -> float: + f = self._inner.cheap_eval(artifact) + return self._apply(f) if self._state.active else f + + def learned_eval(self, artifact: Evolvable) -> Tuple[float, float]: + score, uncertainty = self._inner.learned_eval(artifact) + if not self._state.active: + return score, uncertainty + # Safety floor: never report less uncertainty than the validation + # residual. Audit priority rises with uncertainty, so this cannot + # claim a poorly-fitting boost needs less oracle spend. + return self._apply(score), max(self._resid_sd, uncertainty) + + def eval_counts(self, artifact: Evolvable, + floor: Optional[float] = None) -> Tuple[float, float]: + return self._inner.eval_counts(artifact, floor=floor) + + def full_eval(self, artifact: Evolvable) -> float: + return self._inner.full_eval(artifact) + + def oracle_eval(self, artifact: Evolvable) -> float: # legacy alias + return self._inner.full_eval(artifact) diff --git a/agentdescent/audit/tap.py b/agentdescent/audit/tap.py index 5f05be8d..7471c5cb 100644 --- a/agentdescent/audit/tap.py +++ b/agentdescent/audit/tap.py @@ -89,6 +89,19 @@ def _take_signature() -> str: return signature + +def _resolve_version_extra(extra: Any) -> Any: + """Call a live ``version_extra`` (e.g. ``boost.version_tag``) before fingerprinting. + + ``verifier_fingerprint`` stringifies ``extra`` via ``repr``. A bound method's + ``repr`` does not change with the boost's state, so a callable must be + evaluated first or activation would be invisible to the audit. + """ + if callable(extra): + return extra() + return extra + + class AuditedReward: """A cheap verifier that hands a sampled minority of its work to the truth. @@ -185,8 +198,12 @@ def __init__(self, verifier: Callable[[Any, str], float], *, self.rates = dict(rates or {}) self.calibration_fraction = calibration_fraction self.seed = seed + # Keep the ingredients so a live ``version_extra`` (e.g. a boost's + # ``version_tag``) can be re-fingerprinted when the boost activates. + self._version_extra = version_extra + self._explicit_verifier_version = verifier_version self.verifier_version = verifier_version or verifier_fingerprint( - verifier, extra=version_extra) + verifier, extra=_resolve_version_extra(version_extra)) #: Units seen, and units audited. A sample rate that never fires is the #: quiet failure this makes loud. self._warn_about_the_rates() @@ -197,6 +214,20 @@ def __init__(self, verifier: Callable[[Any, str], float], *, self.skipped = 0 self._lock = threading.Lock() + def refresh_verifier_version(self) -> str: + """Re-derive ``verifier_version`` from a live ``version_extra``. + + Used when an attached :class:`~agentdescent.audit.boost.VerifierBoost` + activates: its ``version_tag`` changes and the audit's fingerprint + (gate, watch, store filter) must follow, or the old calibration keeps + applying to a judge that has moved. + """ + if self._explicit_verifier_version is not None: + return self.verifier_version + self.verifier_version = verifier_fingerprint( + self.verifier, extra=_resolve_version_extra(self._version_extra)) + return self.verifier_version + # -- the Reward contract ------------------------------------------------- def __call__(self, task: Any, output: str) -> float: diff --git a/agentdescent/audit/wiring.py b/agentdescent/audit/wiring.py index 310c0f6a..55ede478 100644 --- a/agentdescent/audit/wiring.py +++ b/agentdescent/audit/wiring.py @@ -71,6 +71,12 @@ class Audit: #: :meth:`~agentdescent.audit.gate.VerifierWatch.check` between rounds, or #: :meth:`~agentdescent.audit.gate.VerifierWatch.on_merge` from a hook. watch: VerifierWatch + #: Optional verifier bootstrap flywheel (see + #: :class:`~agentdescent.audit.boost.VerifierBoost`). ``None`` keeps the + #: old behaviour exactly. When set, :meth:`recompute` retrains it after + #: re-estimating the rectification, so the loop's calibration and the + #: verifier's own improvement move together. + boost: Optional["VerifierBoost"] = None @property def verifier_version(self) -> str: @@ -85,7 +91,23 @@ def rectification(self) -> Rectification: return self.calibrator.current(self.verifier_version) def recompute(self) -> Rectification: - """Re-read the store and re-estimate. Call after a batch resolves.""" + """Re-read the store and re-estimate. Call after a batch resolves. + + When a :class:`~agentdescent.audit.boost.VerifierBoost` is attached, + it is retrained here too -- a batch of resolved labels is exactly the + moment the flywheel should learn from. After a retrain that changes + ``version_tag``, the reward's fingerprint is refreshed and + :meth:`~agentdescent.audit.gate.VerifierWatch.check` runs so the old + calibration is withdrawn before the new version is estimated. + """ + if self.boost is not None: + # Train against labels scored under the version currently in force + # (before activation changes the tag). + self.boost.retrain(verifier_version=self.verifier_version) + prev = self.verifier_version + self.reward.refresh_verifier_version() + if self.reward.verifier_version != prev: + self.watch.check() return self.calibrator.recompute(self.verifier_version) def rebalance(self, unseen: float, **kw: Any) -> float: @@ -173,7 +195,8 @@ def attach(verifier: Callable[[Any, str], float], *, verifier_version: Optional[str] = None, version_extra: Any = None, watch_ids: Iterable[str] = (), - watch_globs: Sequence[str] = ()) -> Audit: + watch_globs: Sequence[str] = (), + boost: Optional["VerifierBoost"] = None) -> Audit: """Assemble the audit around ``verifier`` and return what ``evolve()`` needs. Parameters @@ -228,12 +251,19 @@ def attach(verifier: Callable[[Any, str], float], *, if not policy.enabled: enabled = False + if boost is not None and version_extra is None and verifier_version is None: + # Live tag so activation changes the audit fingerprint (review #273). + version_extra = boost.version_tag + reward = AuditedReward( verifier, oracle=oracle, store=store, draw_by=draw_by, sample_rate=default_rate, stratify=stratify, rates=rates, calibration_fraction=calibration_fraction, seed=seed, verifier_version=verifier_version, version_extra=version_extra) + if boost is not None and boost._version_for_training is None: + boost._version_for_training = lambda: reward.verifier_version + calibrator = Calibrator(store, alpha=policy.alpha if policy else 0.05, seed=seed) acceptance = RectifiedAcceptance( @@ -251,4 +281,4 @@ def attach(verifier: Callable[[Any, str], float], *, watch.check() return Audit(reward=reward, run=RenderTap(run) if run is not None else None, acceptance=acceptance, store=store, calibrator=calibrator, - watch=watch) + watch=watch, boost=boost) diff --git a/changelog.d/273-verifier-boost.md b/changelog.d/273-verifier-boost.md new file mode 100644 index 00000000..dcb7861c --- /dev/null +++ b/changelog.d/273-verifier-boost.md @@ -0,0 +1,14 @@ +Added +Verifier bootstrap flywheel (`agentdescent/audit/boost.py`, #273): +`VerifierBoost` wraps any verifier and learns a monotone calibration curve +(`isotonic_fit`, numpy-only PAV) from the audit's **IMPROVEMENT** pool, then +activates it **only if it provably reduces residual error on a held-out slice +of that pool** (the bootstrap gate). When active it corrects the cheap and +learned layers the aggregator uses to rank candidates and the audit uses for +its trust signal; `eval_counts` / `full_eval` are delegated **raw**, so the +commit gate keeps the oracle's authority over what commits. It trains on the +IMPROVEMENT pool only (never the CALIBRATION pool that measures it), filters +to the current verifier version, and declares itself as a new verifier version +(`version_tag`) so `attach(boost=...)` / `Audit.recompute()` refresh the audit +fingerprint and `VerifierWatch` withdraws the old calibration. Default +behaviour unchanged -- no boost means no fitting and no correction. diff --git a/docs/api.md b/docs/api.md index e8f84dac..89c1f1c4 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1294,6 +1294,7 @@ AuditedReward( |---|---| | `calibration_set() -> list` | Resolved CALIBRATION records for *this* verifier version. | | `pending() -> list` | Audited units still waiting on truth. | +| `refresh_verifier_version() -> str` | Re-derive `verifier_version` from a live `version_extra`. | ### `RenderTap(run: Callable[[str, Any], str]) -> None` @@ -1316,7 +1317,8 @@ Audit( acceptance: RectifiedAcceptance, store: AuditStore, calibrator: Calibrator, - watch: VerifierWatch + watch: VerifierWatch, + boost: Optional['VerifierBoost'] = None ) -> None ``` @@ -1350,7 +1352,8 @@ attach( verifier_version: Optional[str] = None, version_extra: Any = None, watch_ids: Iterable[str] = (), - watch_globs: Sequence[str] = () + watch_globs: Sequence[str] = (), + boost: Optional['VerifierBoost'] = None ) -> Audit ``` @@ -1373,6 +1376,7 @@ attach( | `version_extra` | `Any` | `None` | | | `watch_ids` | `Iterable[str]` | `()` | Artifact ids and diff-key globs that mean the verifier changed. Nothing is watched by default, which is right for a fixed function and exactly wrong for a run that evolves its own judge -- see `VerifierWatch`. | | `watch_globs` | `Sequence[str]` | `()` | As `watch_ids`. | +| `boost` | `Optional['VerifierBoost']` | `None` | | --- diff --git a/tests/test_verifier_boost.py b/tests/test_verifier_boost.py new file mode 100644 index 00000000..e2ab8683 --- /dev/null +++ b/tests/test_verifier_boost.py @@ -0,0 +1,366 @@ +"""The verifier bootstrap flywheel has to stay right in the exact places a +self-improving judge is dangerous. + +A verifier that learns from its own labels could learn to be confident-but-wrong, +so every one of these fixtures pins a safety property: the correction must be +monotone (it ranks), must only activate when it provably reduces residual error +on a held-out slice, must train on the IMPROVEMENT pool only (never the pool +that measures it), must never touch the commit gate, and must declare itself as +a new verifier version so the audit's calibration is withdrawn and re-measured. +""" + +import math + +import numpy as np +import pytest + +from agentdescent.audit import ( + AuditStore, + AuditRecord, + Purpose, + VerifierBoost, + attach, + isotonic_fit, +) + + +# -- helpers ------------------------------------------------------------------- + + +def _record(score, oracle, version="v1", purpose=Purpose.IMPROVEMENT): + return AuditRecord( + record_id=f"r{score}-{oracle}-{purpose.value}", + task_id="t", artifact_signature="sig", output="o", + verifier_version=version, verifier_score=float(score), + inclusion_prob=0.5, purpose=purpose, oracle_score=float(oracle), + resolved_at=1.0) + + +def _store_with(records, path=None): + store = AuditStore(path) + for rec in records: + store.append(rec) + return store + + +def _biased_scores(oracle_values, bias=0.2): + """verifier over-credits every oracle value by ``bias`` (clipped to [0,1]).""" + return [min(1.0, max(0.0, y + bias)) for y in oracle_values] + + +def _noisy_verifier(oracle_values): + """A verifier whose score is unrelated to the oracle: nothing to learn.""" + return [min(1.0, max(0.0, 0.5 + 0.4 * math.sin(i * 1.7))) for i in range(len(oracle_values))] + + +class _FakeVerifier: + """A minimal VerifierProtocol: a fixed score, a fixed uncertainty, raw gate.""" + + def __init__(self, score=0.5, full_counts=(5, 5), uncertainty=0.1): + self.score = score + self.full_counts = full_counts + self.uncertainty = uncertainty + self.full_eval_matches_counts = True + + def rule_eval(self, artifact): + return self.score + + def cheap_eval(self, artifact): + return self.score + + def learned_eval(self, artifact): + return self.score, self.uncertainty + + def eval_counts(self, artifact, floor=None): + return self.full_counts + + def full_eval(self, artifact): + s, f = self.full_counts + return s / (s + f) + + +# -- the correction itself ------------------------------------------------------ + + +def test_isotonic_fit_is_monotone_and_approximates_the_target(): + y = np.linspace(0.1, 0.7, 30) # no clipping under the bias + x = _biased_scores(y, bias=0.2) + xs, vals = isotonic_fit(x, y) + assert (np.diff(vals) >= -1e-9).all() # monotone non-decreasing + # a constant-bias line is fit closely: g(x) ≈ y + err = np.max(np.abs(np.interp(np.asarray(x), xs, vals) - np.asarray(y))) + assert err < 0.05 + + +def test_the_correction_preserves_score_order(): + y = np.array([0.1, 0.2, 0.3, 0.4, 0.5]) + x = _biased_scores(y, bias=0.1) + xs, vals = isotonic_fit(x, y) + grid = np.linspace(0.0, 1.0, 200) + g = np.interp(grid, xs, vals) + assert (np.diff(g) >= -1e-9).all() + assert g[0] >= 0.0 and g[-1] <= 1.0 + + +# -- the bootstrap gate ---------------------------------------------------------- + + +def test_no_labels_leaves_the_boost_inactive(): + store = AuditStore() + boost = VerifierBoost(_FakeVerifier(), store) + assert boost.is_active is False + assert "insufficient improvement labels" in boost.status()["reason"] + + +def test_too_few_labels_stays_inactive(): + store = _store_with([_record(0.5, 0.3)]) + boost = VerifierBoost(_FakeVerifier(score=0.5), store, min_labels=10, + min_val_labels=3) + assert boost.is_active is False + + +def test_a_genuine_bias_is_learned_and_validated(): + y = list(np.linspace(0.1, 0.9, 24)) + s = _biased_scores(y, bias=0.2) + store = _store_with([_record(score, oracle) for score, oracle in zip(s, y)]) + verifier = _FakeVerifier(score=0.5) # biased score for 0.3 truth + boost = VerifierBoost(verifier, store, min_labels=10, min_val_labels=3, + seed=1) + assert boost.is_active is True + status = boost.status() + assert status["error_after"] < status["error_before"] + # the corrected cheap score is closer to the oracle than the raw one + raw, corrected = 0.5, boost.cheap_eval(object()) + assert abs(corrected - 0.3) < abs(raw - 0.3) + + +def test_a_boost_that_does_not_help_is_rejected(): + y = list(np.linspace(0.1, 0.9, 24)) + s = _noisy_verifier(y) # unrelated to truth + store = _store_with([_record(score, oracle) for score, oracle in zip(s, y)]) + boost = VerifierBoost(_FakeVerifier(score=0.5), store, min_labels=10, + min_val_labels=3, seed=1) + assert boost.is_active is False + assert "bootstrap gate" in boost.status()["reason"] + + +def test_min_gain_raises_the_bar(): + # A partially-learnable bias: isotopic can remove the constant part but not + # the noise, so the error reduction is real yet below a 50% bar. + rng = np.random.default_rng(0) + y = np.linspace(0.1, 0.8, 40) + s = np.clip(y + 0.1 + rng.normal(0.0, 0.15, size=y.shape), 0.0, 1.0) + store = _store_with([_record(score, oracle) for score, oracle in zip(s, y)]) + # a lenient gate accepts (any reduction), a strict one refuses (< 50%) + lenient = VerifierBoost(_FakeVerifier(score=0.5), store, min_labels=10, + min_val_labels=3, seed=3, min_gain=0.0) + strict = VerifierBoost(_FakeVerifier(score=0.5), store, min_labels=10, + min_val_labels=3, seed=3, min_gain=0.5) + assert lenient.is_active is True + assert strict.is_active is False + + +# -- what the boost may and may not touch --------------------------------------- + + +def test_the_commit_gate_is_delegated_raw_when_active(): + y = list(np.linspace(0.1, 0.9, 24)) + s = _biased_scores(y, bias=0.2) + store = _store_with([_record(score, oracle) for score, oracle in zip(s, y)]) + boost = VerifierBoost(_FakeVerifier(score=0.5, full_counts=(4, 4)), store, + min_labels=10, min_val_labels=3, seed=1) + assert boost.is_active is True + assert boost.eval_counts(object()) == (4, 4) # raw + assert boost.full_eval(object()) == 0.5 # raw + assert boost.rule_eval(object()) == 0.5 # raw + + +def test_the_learned_uncertainty_reflects_the_validation_residual(): + y = list(np.linspace(0.1, 0.9, 24)) + s = _biased_scores(y, bias=0.2) + store = _store_with([_record(score, oracle) for score, oracle in zip(s, y)]) + boost = VerifierBoost(_FakeVerifier(score=0.5, uncertainty=0.1), store, + min_labels=10, min_val_labels=3, seed=1) + _, uncertainty = boost.learned_eval(object()) + assert uncertainty >= boost.status()["resid_sd"] + assert uncertainty >= 0.1 + + +def test_training_uses_the_improvement_pool_only(): + # A CALIBRATION record -- which exists to *measure* the verifier, not to + # change it -- with a wild outlier must not enter the fit at all. + y = list(np.linspace(0.1, 0.9, 24)) + s = _biased_scores(y, bias=0.2) + records = [_record(score, oracle) for score, oracle in zip(s, y)] + records.append(_record(0.5, 1.0, purpose=Purpose.CALIBRATION)) + store = _store_with(records) + boost = VerifierBoost(_FakeVerifier(score=0.5), store, min_labels=10, + min_val_labels=3, seed=1) + # the calibration record never trained anything: without it the fit is the + # same -- and the corrected score is still the same clean 0.3 + assert abs(boost.cheap_eval(object()) - 0.3) < abs(0.5 - 0.3) + + +# -- versioning ----------------------------------------------------------------- + + +def test_version_tag_changes_on_activation_and_is_stable_while_active(): + y = list(np.linspace(0.1, 0.9, 24)) + s = _biased_scores(y, bias=0.2) + store = _store_with([_record(score, oracle) for score, oracle in zip(s, y)]) + boost = VerifierBoost(_FakeVerifier(score=0.5), store, min_labels=10, + min_val_labels=3, seed=1, auto=False) + assert boost.version_tag() == "boost:inactive" + boost.retrain() + tag = boost.version_tag() + assert tag.startswith("boost:active:") + assert boost.version_tag() == tag # stable between fits + + +# -- wiring --------------------------------------------------------------------- + + +def test_audit_recompute_retrains_an_attached_boost(): + store = AuditStore() + boost = VerifierBoost(_FakeVerifier(), store, auto=False) + audit = attach(verifier=lambda task, output: 0.5, + oracle=lambda task, output: 0.7, + store=store, boost=boost) + assert boost.status()["fitted"] is False + audit.recompute() # no labels -> retrain runs, inactive + assert boost.status()["fitted"] is True + + +def test_no_boost_means_recompute_is_unchanged(): + store = AuditStore() + audit = attach(verifier=lambda task, output: 0.5, + oracle=lambda task, output: 0.7, store=store) + rect = audit.recompute() # must not raise + assert rect is not None + + +def test_the_boost_is_a_valid_engine_verifier(tmp_path): + # The wrapper must satisfy VerifierProtocol so evolve() can consume it + # through Policies(verifier=...). Empty store -> inactive -> raw path. + from agentdescent import Policies, ThreeLayerVerifier + from agentdescent.evolution import SingleSlot, Task, evolve + + tasks = [Task(id=f"t{i}", prompt=f"item {i}") for i in range(8)] + + def run(rendered, task): + return "answer" + (" 2026" if "year" in rendered else "") + + def reward(task, output): + return 1.0 if "2026" in output else 0.0 + + verifier = ThreeLayerVerifier(eval_fn=lambda a, ts: a.score(ts), + held_out=tasks, rule_subset=4) + boost = VerifierBoost(verifier, AuditStore()) # no labels -> inactive + result = evolve( + tasks, reward, run=run, + propose=lambda r, t, o, re: "always state the year", + strategy=SingleSlot(initial_value="seed"), rounds=2, n_workers=2, + repo_path=str(tmp_path / "repo"), policies=Policies(verifier=boost)) + assert boost.is_active is False + assert result.outcomes().get("committed", 0) >= 1 + + +# -- review #273 regressions ---------------------------------------------------- + + +def test_empty_train_split_does_not_crash(): + """min_labels > min_val_labels is required; equal values refuse at init + (review #273 blocking item 3).""" + store = _store_with([_record(0.5, 0.3) for _ in range(10)]) + with pytest.raises(ValueError, match="min_labels"): + VerifierBoost(_FakeVerifier(), store, min_labels=10, min_val_labels=10) + + +def test_training_filters_to_the_current_verifier_version(): + """Labels from an older verifier must not train the current judge + (review #273 blocking item 2).""" + # 200 old biased labels + 200 new unbiased labels under different versions. + old = [_record(min(1.0, y + 0.4), y, version="old") + for y in list(__import__("numpy").linspace(0.1, 0.9, 200))] + new = [_record(y, y, version="new") + for y in list(__import__("numpy").linspace(0.1, 0.9, 200))] + store = _store_with(old + new) + # Train only on "new": unbiased data should not activate a correction that + # pulls 0.5 toward the old bias. + boost = VerifierBoost( + _FakeVerifier(score=0.5), store, min_labels=50, min_val_labels=10, + seed=1, min_gain=0.0, auto=False, + version_for_training=lambda: "new") + boost.retrain() + # With only unbiased new labels there is nothing useful to learn; either + # inactive or a near-identity curve. Critically, g(0.5) must not be the + # old-bias correction (~0.1). + if boost.is_active: + assert abs(boost.cheap_eval(object()) - 0.5) < 0.15 + + +def test_audit_version_changes_when_boost_activates(tmp_path): + """attach + recompute must refresh verifier_version on activation so + VerifierWatch withdraws the old calibration (review #273 blocking item 1).""" + y = list(__import__("numpy").linspace(0.1, 0.9, 80)) + s = _biased_scores(y, bias=0.2) + store = _store_with( + [_record(score, oracle, version="will-refresh") + for score, oracle in zip(s, y)], + path=str(tmp_path / "audit.jsonl")) + # Pre-seed records under a fingerprint we will replace; attach will set a + # live version_extra from the boost tag. Use an explicit store and boost. + boost = VerifierBoost(_FakeVerifier(score=0.5), store, min_labels=20, + min_val_labels=5, seed=1, min_gain=0.0, auto=False) + audit = attach(verifier=lambda task, output: 0.5, + oracle=lambda task, output: 0.3, + store=store, boost=boost) + before = audit.verifier_version + assert boost.version_tag() == "boost:inactive" + # Retrain against whatever version the reward currently reports; seed the + # store with records under that version so the filter finds them. + current = audit.verifier_version + for score, oracle in zip(s, y): + store.append(_record(score, oracle, version=current)) + audit.recompute() + assert boost.is_active is True + after = audit.verifier_version + assert after != before, ( + "activation must change the audit fingerprint so the old calibration " + f"is withdrawn; before={before} after={after}") + assert boost.version_tag().startswith("boost:active:") + + +def test_wrapper_preserves_legacy_false_full_eval_declaration(): + """A wrapped verifier with oracle_shares_full_set=False must stay + independent after wrapping (review #273 P1).""" + class Legacy: + oracle_shares_full_set = False + def rule_eval(self, a): return 0.5 + def cheap_eval(self, a): return 0.5 + def learned_eval(self, a): return 0.5, 0.1 + def eval_counts(self, a, floor=None): return (5, 5) + def full_eval(self, a): return 0.4 # independent + + from agentdescent.verifier import shares_eval_counts + inner = Legacy() + assert shares_eval_counts(inner) is False + boost = VerifierBoost(inner, AuditStore(), auto=False) + assert shares_eval_counts(boost) is False + assert boost.full_eval_matches_counts is False + + +def test_wrapper_preserves_missing_full_eval_declaration(): + """No declaration on the inner verifier => independent measurements.""" + class Bare: + def rule_eval(self, a): return 0.5 + def cheap_eval(self, a): return 0.5 + def learned_eval(self, a): return 0.5, 0.1 + def eval_counts(self, a, floor=None): return (5, 5) + def full_eval(self, a): return 0.5 + + from agentdescent.verifier import shares_eval_counts + inner = Bare() + assert shares_eval_counts(inner) is False + boost = VerifierBoost(inner, AuditStore(), auto=False) + assert shares_eval_counts(boost) is False