audit: VerifierBoost bootstrap gate must beat a constant-mean baseline (#306)

agentdescent/audit/boost.py

@@ -138,6 +138,10 @@ class BoostState:
     error_after: float
     reason: str
     fit_at: float
+    #: Validation MSE of the best *constant* predictor (the train slice's mean
+    #: oracle score). ``inf`` -- also the default, so older constructors keep
+    #: working -- means "not measured".
+    error_const: float = math.inf
 
 
 class VerifierBoost:
@@ -253,7 +257,31 @@ def _apply_candidate(f: float) -> float:
                             [r.oracle_score for r in val])
         error_after = _mse([_apply_candidate(r.verifier_score) for r in val],
                            [r.oracle_score for r in val])
-        if error_after <= error_before * (1.0 - self.min_gain):
+        # A curve that only shifts the verifier's mean passes the raw-verifier
+        # test above while carrying no ranking information at all: it must also
+        # beat the best constant predictor. (A constant offset is the audit
+        # ``Calibrator``'s job -- ``delta_hat`` -- not the ranking layer's.)
+        const = float(np.mean(ys))
+        error_const = _mse([const] * len(val), [r.oracle_score for r in val])
+        beats_raw = error_after <= error_before * (1.0 - self.min_gain)
+        beats_const = error_after <= error_const * (1.0 - self.min_gain)
+        if beats_raw and not beats_const:
+            return self._deactivate(
+                f"no discrimination beyond a constant: corrected error "
+                f"{error_after:.4f} does not beat the constant-mean baseline "
+                f"{error_const:.4f} by min_gain {self.min_gain} (raw "
+                f"{error_before:.4f}); a pure bias shift is the Calibrator's "
+                f"job, not the ranking layer's",
+                n_train=len(train), n_val=len(val),
+                error_before=error_before, error_after=error_after,
+                error_const=error_const)
+        if beats_raw and np.unique(fit_vals).size < 2:
+            return self._deactivate(
+                "degenerate calibration curve: fewer than 2 distinct fitted levels",
+                n_train=len(train), n_val=len(val),
+                error_before=error_before, error_after=error_after,
+                error_const=error_const)
+        if beats_raw:
             digest = hashlib.sha256(json.dumps({
                 "xs": fit_xs.tolist(), "vals": fit_vals.tolist(),
                 "n_train": len(train)}, sort_keys=True).encode()).hexdigest()
@@ -265,19 +293,41 @@ def _apply_candidate(f: float) -> float:
             self._resid_sd = math.sqrt(error_after)
             self._state = BoostState(True, True, len(train), len(val),
                                      error_before, error_after,
-                                     "validated", time.time())
+                                     "validated", time.time(), error_const)
             return self._state
         return self._deactivate(
             f"bootstrap gate: corrected error {error_after:.4f} is not below "
-            f"raw {error_before:.4f} by min_gain {self.min_gain}")
+            f"raw {error_before:.4f} by min_gain {self.min_gain}",
+            n_train=len(train), n_val=len(val),
+            error_before=error_before, error_after=error_after,
+            error_const=error_const)
 
-    def _deactivate(self, reason: str) -> BoostState:
+    def _deactivate(
+        self,
+        reason: str,
+        *,
+        n_train: int = 0,
+        n_val: int = 0,
+        error_before: float = math.inf,
+        error_after: float = math.inf,
+        error_const: float = math.inf,
+    ) -> BoostState:
+        """Record a rejected fit. Preserve measured diagnostics when known.
+
+        Call sites that already computed validation MSE (and the constant
+        baseline) must pass them through so ``status()`` / ``retrain()`` report
+        the numbers that justified the rejection rather than the default
+        ``inf`` ("not measured").
+        """
         self._xs = None
         self._vals = None
         self._fit_hash = ""
         self._resid_sd = math.inf
-        self._state = BoostState(False, True, 0, 0, math.inf, math.inf,
-                                 reason, time.time())
+        self._state = BoostState(
+            False, True, int(n_train), int(n_val),
+            float(error_before), float(error_after),
+            reason, time.time(), float(error_const),
+        )
         return self._state
 
     @property
@@ -312,6 +362,7 @@ def status(self) -> dict:
             "n_val": s.n_val,
             "error_before": s.error_before,
             "error_after": s.error_after,
+            "error_const": s.error_const,
             "resid_sd": self._resid_sd,
             "reason": s.reason,
             "fit_at": s.fit_at,

changelog.d/306-boost-constant-baseline.md

@@ -0,0 +1,9 @@
+Fixed
+`VerifierBoost.retrain()` no longer activates a calibration curve that
+only shifts the verifier's mean: the bootstrap gate must now also beat the best
+constant predictor (the train slice's mean oracle score) on the validation
+slice by `min_gain`, so a verifier with no discriminating power that merely
+reads systematically high or low is rejected ("no discrimination beyond a
+constant") instead of being "validated" and re-ranking candidates. A curve with
+fewer than two distinct fitted levels is rejected too, and `status()` and
+`BoostState` expose the new `error_const` baseline (#306). On rejection via the constant-baseline or bootstrap gate, the measured `error_const` / validation MSEs are retained on the rejected `BoostState` so `status()` can diagnose why the candidate failed.

tests/test_verifier_boost.py

@@ -364,3 +364,96 @@ def full_eval(self, a): return 0.5
     assert shares_eval_counts(inner) is False
     boost = VerifierBoost(inner, AuditStore(), auto=False)
     assert shares_eval_counts(boost) is False
+
+
+# -- the gate must beat a constant predictor (#306) ------------------------------
+
+
+class _ListStore:
+    """The slice of AuditStore the boost reads: just ``for_improvement``."""
+
+    def __init__(self, recs):
+        self._recs = recs
+
+    def for_improvement(self, version=None):
+        return self._recs
+
+
+def _namespace_records(pairs):
+    import types
+    return [types.SimpleNamespace(resolved=True, verifier_score=f, oracle_score=y)
+            for f, y in pairs]
+
+
+def test_a_pure_bias_shift_without_discrimination_is_not_validated():
+    """Score = noise in [0.8, 1.0], oracle passes 30% of the time, uncorrelated.
+
+    The raw verifier is badly biased (MSE ~0.58), so *any* curve that moves the
+    mean down beats it -- yet the curve carries no ranking information and is
+    worse than predicting the training mean. The gate must say so."""
+    import random
+
+    rng = random.Random(1)
+    recs = _namespace_records(
+        (0.8 + 0.2 * rng.random(), 1.0 if rng.random() < 0.3 else 0.0)
+        for _ in range(400))
+    boost = VerifierBoost(_FakeVerifier(score=0.9), _ListStore(recs))
+    status = boost.status()
+    assert boost.is_active is False
+    assert "constant" in status["reason"]
+    assert boost.version_tag() == "boost:inactive"
+    # rejection still exposes the measured baseline (and the MSEs that
+    # justified it) rather than the default "not measured" inf
+    assert math.isfinite(status["error_const"]) and status["error_const"] > 0.0
+    assert math.isfinite(status["error_before"])
+    assert math.isfinite(status["error_after"])
+    assert status["n_train"] > 0 and status["n_val"] > 0
+    # the curve is not applied to the ranking layers
+    assert boost.cheap_eval(object()) == 0.9
+
+
+def test_a_biased_but_informative_verifier_still_activates():
+    """Control: monotone-informative yet miscalibrated (oracle = 1 iff f > 0.9,
+    scores in [0.8, 1.0]) is the case the boost exists for."""
+    import random
+
+    rng = random.Random(2)
+    pairs = []
+    for _ in range(400):
+        f = 0.8 + 0.2 * rng.random()
+        pairs.append((f, 1.0 if f > 0.9 else 0.0))
+    boost = VerifierBoost(_FakeVerifier(score=0.95), _ListStore(_namespace_records(pairs)))
+    status = boost.status()
+    assert boost.is_active is True, status["reason"]
+    assert status["error_after"] <= status["error_const"] * (1.0 - boost.min_gain)
+    assert status["error_after"] <= status["error_before"] * (1.0 - boost.min_gain)
+    assert boost.version_tag().startswith("boost:active:")
+    assert boost.cheap_eval(object()) > 0.5      # f=0.95 is in the passing region
+
+
+def test_status_exposes_the_constant_baseline_and_keeps_existing_keys():
+    y = list(np.linspace(0.1, 0.9, 24))
+    s = _biased_scores(y, bias=0.2)
+    store = _store_with([_record(score, oracle) for score, oracle in zip(s, y)])
+    boost = VerifierBoost(_FakeVerifier(score=0.5), store, min_labels=10,
+                          min_val_labels=3, seed=1)
+    status = boost.status()
+    assert {"active", "fitted", "n_train", "n_val", "error_before", "error_after",
+            "resid_sd", "reason", "fit_at", "version_tag"} <= set(status)
+    assert math.isfinite(status["error_const"]) and status["error_const"] > 0.0
+    # a state built the old way (no baseline) reports "not measured"
+    from agentdescent.audit import boost as boost_mod
+    old = boost_mod.BoostState(False, False, 0, 0, math.inf, math.inf, "x", 0.0)
+    assert old.error_const == math.inf
+    # before any fit (or after an early rejection with no measured baseline)
+    # the baseline is "not measured"
+    fresh = VerifierBoost(_FakeVerifier(), _store_with([]), auto=False)
+    assert fresh.status()["error_const"] == math.inf
+
+
+def test_a_degenerate_single_level_curve_is_rejected_with_a_reason():
+    # min_gain=0 lets a one-level (constant) curve tie the constant baseline.
+    recs = _namespace_records([(0.5 + 0.001 * i, 0.5) for i in range(100)])
+    boost = VerifierBoost(_FakeVerifier(), _ListStore(recs), min_gain=0.0)
+    assert boost.is_active is False
+    assert "degenerate" in boost.status()["reason"]