policy: rank levers from what previous runs measured

gitm/agents/policy.py

@@ -1,11 +1,18 @@
-"""Selection policy: pre-filter by safety, rank by predicted delta, return top-N."""
+"""Selection policy: pre-filter by safety, rank by predicted delta, return top-N.
+
+Ranking is a precedence tuple rather than one blended number, for the reason
+:mod:`gitm.playbook.match` gives: terms answering different questions should not
+be collapsed into a scalar where one can quietly outvote another. Gate first,
+then evidence quality, then magnitude, then a deterministic tie-break.
+"""
 
 from __future__ import annotations
 
 from collections.abc import Iterable
 from dataclasses import dataclass
 
 from gitm.kernels.spec import InterventionSpec
+from gitm.optimizer.history import History, record_for
 from gitm.optimizer.preconditions import GateContext, applicable
 from gitm.optimizer.replay import predict_delta
 from gitm.tracer.schema import Trace
@@ -16,6 +23,17 @@ class RankedCandidate:
     spec: InterventionSpec
     predicted_delta: float
     rejected_reason: str | None = None
+    #: Where ``predicted_delta``'s effect estimate came from: ``"prior"`` for
+    #: the spec's hand-authored ``expected_delta_mean``, ``"measured"`` for a
+    #: delta this lever recorded on this GPU. Carried for the same reason
+    #: ``rejected_reason`` is: a number is worth less without what produced it.
+    delta_source: str = "prior"
+    #: Ranked below every undemoted candidate, but never removed. A lever whose
+    #: record both won and lost has not come out neutral — it behaved differently
+    #: under conditions the record does not capture, so it is the weaker bet
+    #: while that holds. The demotion lifts by itself once the record stops
+    #: disagreeing: it describes the evidence, not the lever.
+    demoted: bool = False
 
 
 @dataclass
@@ -24,6 +42,11 @@ class Policy:
 
     require_qualification_commit: bool = False
     skip_high_risk: bool = False
+    #: Score a lever from what it measured before, where there is a record for
+    #: this GPU. Off by default because it changes which experiments run, and
+    #: that should be a decision someone made rather than one that arrived
+    #: with an upgrade.
+    use_history: bool = False
 
 
 def select_interventions(
@@ -33,7 +56,21 @@ def select_interventions(
     top_n: int = 5,
     *,
     ctx: GateContext | None = None,
+    history: History | None = None,
+    gpu_sku: str | None = None,
+    fingerprint: str | None = None,
 ) -> list[RankedCandidate]:
+    """Rank the library for this trace, rejected candidates last.
+
+    ``history`` is passed in rather than read from disk here, so ranking stays a
+    pure function of what it is given and a caller can rank against a record it
+    has already filtered. It is consulted only when ``policy.use_history`` is on
+    *and* ``gpu_sku`` names the box: a result measured on an H100 says nothing
+    about an MI355X, and scoring one from the other is the mistake the record's
+    GPU key exists to prevent. No SKU therefore means no substitution, not a
+    guess at which box the record came from.
+    """
+    use_history = policy.use_history and history is not None and gpu_sku is not None
     candidates: list[RankedCandidate] = []
 
     for spec in library:
@@ -46,10 +83,43 @@ def select_interventions(
             reason = "policy.skip_high_risk"
         elif reason is None and (spec.safety.requires_qualification_commit and not policy.require_qualification_commit):
             reason = "safety.requires_qualification_commit"
-        delta = predict_delta(trace, spec) if reason is None else 0.0
-        candidates.append(RankedCandidate(spec=spec, predicted_delta=delta, rejected_reason=reason))
+        record = (
+            record_for(history, spec.name, gpu_sku=gpu_sku, fingerprint=fingerprint)
+            if use_history and reason is None
+            else None
+        )
+        # A record with no usable delta is still a record: it says the lever was
+        # tried and how it fared, but carries no number to rank on. The prior
+        # stands in that case and only the demotion applies.
+        measured = record.mean_delta if record is not None else None
+        delta = predict_delta(trace, spec, delta_mean=measured) if reason is None else 0.0
+        candidates.append(RankedCandidate(
+            spec=spec,
+            predicted_delta=delta,
+            rejected_reason=reason,
+            delta_source="measured" if measured is not None else "prior",
+            demoted=bool(record is not None and record.conflicted),
+        ))
 
+    # Four terms, in this order and for these reasons:
+    #
+    # 1. Rejected. The gate's answer is categorical and comes first.
+    # 2. Not worth a run. A lever whose estimate is zero or negative is not a
+    #    candidate whatever the evidence behind it says, so it sorts below every
+    #    lever that might help. This sits above the demotion because a lever
+    #    measured at -9% every time is a worse bet than one that is merely
+    #    inconsistent, and ranking the known loser higher would spend the run on
+    #    a result already in hand.
+    # 3. Demoted. Among levers that might help, prefer the one whose record does
+    #    not disagree with itself.
+    # 4. Magnitude, then name for a deterministic order.
     candidates.sort(
-        key=lambda c: (c.rejected_reason is not None, -c.predicted_delta, c.spec.name)
+        key=lambda c: (
+            c.rejected_reason is not None,
+            c.predicted_delta <= 0.0,
+            c.demoted,
+            -c.predicted_delta,
+            c.spec.name,
+        )
     )
     return candidates[:top_n]

gitm/api.py

@@ -20,6 +20,7 @@ def optimize(
     target: float = 0.15,
     scratch: str | None = None,
     workload_runner: Callable[[], dict[str, Any]] | None = None,
+    use_history: bool | None = None,
 ) -> dict[str, Any]:
     """Run the autonomous 24-hour optimization loop and return a report.
 
@@ -29,6 +30,12 @@ def optimize(
     of ``target`` fraction improvement within ``budget`` wall time, or a
     qualification-gate diagnostic explaining why the floor was not committed.
 
+    ``use_history`` ranks candidates from what previous runs measured on this
+    GPU rather than from the library's hand-authored estimates. It is never
+    asked for here — this entry point does not touch stdin, so an embedded
+    caller cannot be blocked by a prompt it did not expect. ``gitm run`` puts
+    the question to the operator and passes the answer down.
+
     ``workload_runner`` optionally supplies an explicit zero-arg callable that
     launches the workload's GPU work; it runs inside the capture window. When
     omitted, the loop resolves ``workload`` against the registry in
@@ -41,5 +48,6 @@ def optimize(
         target=target,
         scratch=scratch,
         workload_runner=workload_runner,
+        use_history=use_history,
     )
     return run_loop(cfg)

gitm/cli.py

@@ -6,6 +6,7 @@
 import json
 import sys
 from pathlib import Path
+from typing import Any
 
 from gitm.optimizer.deviation import add_deviate_arguments
 from gitm.planner.registry import add_plan_arguments
@@ -84,6 +85,15 @@ def _parser() -> argparse.ArgumentParser:
     run = sub.add_parser("run", help="Run the autonomous optimization loop.")
     run.add_argument("--workload", required=True, help="Workload identifier, e.g. vllm-decode.")
     run.add_argument("--budget", default="24h", help="Wall-clock budget, e.g. 24h.")
+    hist = run.add_mutually_exclusive_group()
+    hist.add_argument(
+        "--use-history", dest="use_history", action="store_true", default=None,
+        help="Rank levers from what previous runs measured on this GPU, without asking.",
+    )
+    hist.add_argument(
+        "--no-history", dest="use_history", action="store_false",
+        help="Ignore previous runs' results and score from the catalog. Deletes nothing.",
+    )
     run.add_argument(
         "--target",
         default="15%",
@@ -264,6 +274,67 @@ def _parse_target(s: str) -> float:
 _HFT_WORKLOADS = {"hft", "hft-lob"}
 
 
+def _ask_use_history(n_runs: int, *, timeout_s: float = 60.0, stream: Any = None,
+                     tty: bool | None = None) -> bool:
+    """Ask whether this run should be scored from what previous runs measured.
+
+    Lives here, and not in the loop, because a prompt is a property of being run
+    by a person at a terminal. ``gitm.optimize`` never touches stdin, so an
+    embedded caller cannot be blocked by a question it did not ask for.
+
+    No answer means yes, for two reasons. An unattended run must not sit on a
+    prompt forever, and of the two answers using the record is the one that
+    discards nothing: declining only skips it for this run. Nothing is deleted
+    either way, since every run writes into its own ``runs/<uuid4>/`` and never
+    touches another run's export.
+
+    Without a terminal there is nobody to ask, so it takes the same default at
+    once rather than waiting out the timeout against a pipe that will not reply.
+    """
+    import select
+
+    stream = sys.stdin if stream is None else stream
+    interactive = tty if tty is not None else bool(getattr(stream, "isatty", lambda: False)())
+    if not interactive:
+        return True
+
+    print(
+        f"\n{n_runs} previous run(s) left measured results."
+        "\n  [Y] rank this run from them   [n] ignore them and score from the catalog"
+        f"\n  Nothing is deleted either way. No answer within {timeout_s:.0f}s uses them."
+        "\n> ",
+        end="", flush=True,
+    )
+    try:
+        ready, _, _ = select.select([stream], [], [], timeout_s)
+    except (OSError, ValueError):  # not a selectable stream
+        return True
+    if not ready:
+        print(f"\n  no answer in {timeout_s:.0f}s \u2014 using previous results.")
+        return True
+    answer = (stream.readline() or "").strip().lower()
+    if answer.startswith("n"):
+        print("  ignoring previous results for this run; they stay on disk.")
+        return False
+    return True
+
+
+def _resolve_use_history(args: Any) -> bool:
+    """What ``--use-history`` said, or the answer to the prompt.
+
+    An explicit flag is never second-guessed, which is what keeps scripted and
+    scheduled runs deterministic. With no flag and no previous results there is
+    nothing to ask about and nothing to rank from.
+    """
+    if getattr(args, "use_history", None) is not None:
+        return bool(args.use_history)
+    from gitm._paths import runs_dir
+    from gitm.optimizer.history import runs_with_results
+
+    n = runs_with_results(runs_dir(args.scratch))
+    return _ask_use_history(n) if n else False
+
+
 def _apply_hft_run_flags(args) -> None:
     """Map the hft-only run flags onto the ``GITM_BENCH_*`` env the workload
     factory reads. Errors if they're used with a non-hft workload, where they
@@ -354,11 +425,14 @@ def main(argv: list[str] | None = None) -> int:
         from gitm import optimize
 
         _apply_hft_run_flags(args)
+        # Asked here, before the loop starts any capture, so nobody answers a
+        # prompt that arrived an hour into a 24h run.
         result = optimize(
             workload=args.workload,
             budget=args.budget,
             target=_parse_target(args.target),
             scratch=args.scratch,
+            use_history=_resolve_use_history(args),
         )
         summary = result.get("summary", {})
         if args.report is not None: