From 16bf37ac8723d7da65e69f384fadce53350395dc Mon Sep 17 00:00:00 2001 From: di-omics Date: Tue, 28 Jul 2026 03:07:18 -0700 Subject: [PATCH 1/2] Compose vision with gates, and cover what a workcell needs to keep running Four layers for the part of an autonomous lab that starts once more than one instrument is involved: whether anything would catch a failure, whether the instruments are still entitled to be trusted, whether an expert has taught the operation, and whether a control loop can close. coverage is the composition and the reason the other three are worth having. vision knows what a camera can and can never see, qc knows which gates exist and whether they fire, and recovery knows which failures destroy material. Nothing joined them, so nothing could answer the question a lab actually asks: for every failure that destroys material, is there anything at all that would catch it? On the reference protocol the answer is 14 failure modes, none covered, 9 of them destructive. The useful part is that they do not split into one problem. Six are visible and their checks are declared and undeployable -- a CV project with a known payoff, and bead_pellet_aspirated is among them. One is invisible but already sits behind a declared gate that cannot fire because the reader is broken, so no camera is involved and a repair closes it. Seven are invisible with no assay reading the quantity they change, and no capability upgrade moves them at any price. Confusing those three is how a CV budget gets spent against a problem no model solves. sota_lift therefore returns both lists, because a report showing only what a capability lifts is a purchase justification. Two things refused as coverage: a vision check that exists but whose requirements are unmet, and a gate that exists but cannot be evaluated. Counting either reports a lab as covered while material is quietly destroyed. durability tracks entitlement rather than predicting failure. It computes whether a planned campaign crosses a service or calibration boundary mid-run, because a plate that started before an expiry and finished after it has an ambiguous provenance and nothing downstream repairs that. No MTBF and no remaining-life estimate: that needs population reliability data this package does not have, and the invented figure becomes a specification. teaching models the transfer. A demonstration is data, not authority -- a scientist doing something twice is two observations with a spread, and the spread is the information. An envelope refuses a tolerance from one demonstration and a machine with one good run reports as indistinguishable from unmeasured. The operations with no envelope are the real backlog of an automation programme and demonstration_queue ranks them the way unlocks() ranks decoding work. feedback asks whether a loop closes, with latency as the load-bearing check: a sensor downstream of its own actuator is a post-mortem wearing the costume of a control loop. No gain, no settling time -- that needs a plant model and measured response data. What it does report is how many plates are already committed between sensor and actuator. Each module was built, adversarially audited on three lenses, and fixed. Includes a fix for attainment() naming the untaught operation in its own refusal, which is the common case and was reading "no expert has demonstrated 'unknown'". 156 new tests, 284 total. --- README.md | 84 +++- autonomous_lab/__init__.py | 120 ++++- autonomous_lab/cli.py | 170 +++++++ autonomous_lab/coverage.py | 749 ++++++++++++++++++++++++++++ autonomous_lab/durability.py | 935 +++++++++++++++++++++++++++++++++++ autonomous_lab/feedback.py | 887 +++++++++++++++++++++++++++++++++ autonomous_lab/teaching.py | 914 ++++++++++++++++++++++++++++++++++ tests/test_coverage.py | 495 +++++++++++++++++++ tests/test_durability.py | 464 +++++++++++++++++ tests/test_feedback.py | 619 +++++++++++++++++++++++ tests/test_teaching.py | 468 ++++++++++++++++++ 11 files changed, 5900 insertions(+), 5 deletions(-) create mode 100644 autonomous_lab/coverage.py create mode 100644 autonomous_lab/durability.py create mode 100644 autonomous_lab/feedback.py create mode 100644 autonomous_lab/teaching.py create mode 100644 tests/test_coverage.py create mode 100644 tests/test_durability.py create mode 100644 tests/test_feedback.py create mode 100644 tests/test_teaching.py diff --git a/README.md b/README.md index c68eac8..cc0b7ef 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,11 @@ autonomous-lab throughput single_cell_genomics # plates/day, or why that number autonomous-lab provenance single_cell_genomics # what could be proven about a run afterwards autonomous-lab lineage single_cell_genomics # which cell did this read come from? autonomous-lab knowledge # encoded expert judgment and robot benchmarks + +autonomous-lab coverage single_cell_genomics # would anything catch each failure? +autonomous-lab durability # what each instrument may be trusted with +autonomous-lab teaching single_cell_genomics # what an expert demonstrated, what the machine attains +autonomous-lab feedback single_cell_genomics # can a control loop actually close? ``` ## What it reports today @@ -318,6 +323,83 @@ adequacy. The STAR's dry motion is validated; its volumetric accuracy at the wor has never been measured, and no camera retires that benchmark. It needs a calibration experiment. +## Would anything catch it? Vision and gates, composed + +Three layers here each hold a third of one answer and never meet. `vision` knows what a +camera can and can never see. `qc` knows which gates exist and whether they can fire. +`recovery` knows which failures destroy material. Nobody joined them, so nobody could answer +the question a lab actually asks: **for every failure that destroys material, is there +anything at all that would catch it?** + +``` +$ autonomous-lab coverage single_cell_genomics + + failure modes 14 + covered 0 + uncovered 14, of which 9 destroy material + closable as it stands 7 (build the detector, or make the gate fire) + needs a new instrument 7 +``` + +The composition is the product, because vision and gates are **complementary instruments, +not alternatives**. Where the photons are identical either way, no model resolves it and +only an assay does. So the report splits three ways, and each third is a different job: + +- **six a camera would catch** -- `bead_pellet_aspirated` among them, the failure the + recovery layer already calls the worst silent one. The check is declared and undeployable: + missing camera, pose, labels, model, validation. That is a CV project with a known payoff. +- **one invisible, but a declared gate already reads it** -- `enzyme_activity_lost` sits + behind `library_quant_before_flow_cell`, which is unsatisfiable because the reader is + broken. No camera is involved. Repair the instrument and the coverage arrives. +- **seven no camera reaches at any capability** -- these are `mandatory_gates()`. Buying a + better model buys nothing against them. They need an assay that does not currently exist + in the protocol. + +`sota_lift(current, proposed)` prices a model upgrade honestly. It returns both lists, and +the second is required output: **a report showing only what a capability lifts is a purchase +justification**, and every purchase justification is correct about the failures it lists and +silent about the ones that make the purchase insufficient. + +Two things it refuses to count as coverage: a vision check that exists but whose +requirements are unmet, and a gate that exists but cannot be evaluated. Both are the +vacuous-pass failure in a new costume -- a plan that counts unbuilt detectors and unfirable +gates reports a lab as covered while material is quietly destroyed. + +## Keeping it running, teaching it, and closing the loop + +Three further layers cover what a workcell needs once it has to survive contact with a +calendar. + +**`durability`** asks what an instrument is currently *entitled* to be trusted with. It is +deliberately not a failure predictor -- that needs population reliability data this package +does not have, and an invented MTBF becomes a specification nobody measured. What it does +compute is whether a planned campaign **crosses a service or calibration boundary mid-run**, +which is the insurance question: a plate that started before an expiry and finished after it +has an ambiguous provenance, and nothing downstream repairs that. An instrument with no +service history reports as unmeasured rather than healthy, matching how an unbenchmarked +operation is untrusted by default. + +**`teaching`** models the transfer that makes any of this worth doing: an expert +demonstrating an operation, and a machine measured against that demonstration. The honest +core is that **a demonstration is data, not authority.** A scientist doing something twice is +not a specification -- it is two observations with a spread, and the spread is the +information. So an envelope refuses to produce a tolerance from a single demonstration, and +a machine with one good run reports as indistinguishable from unmeasured rather than as +meeting the bar. Most operations have no envelope at all, and that list is the real backlog +of an automation programme; `demonstration_queue` ranks what an expert should demonstrate +next by how many operations it unblocks, the same way `unlocks()` ranks decoding work. + +**`feedback`** asks whether a control loop can actually close. Each of measure, compare, and +act has its own way of being fake, and the load-bearing one is latency: **a measurement taken +after the material is consumed cannot steer anything, however accurate it is.** It is a +post-mortem wearing the costume of a control loop. So a loop declares where it senses and +where it corrects, and any loop whose sensor sits downstream of its actuator is refused -- +pure graph reasoning over the protocol, and it kills most proposed closed-loop designs. What +it will not do is model gain, overshoot, or settling, because that needs a plant model and +measured response data nothing here has. What it *can* say is how many plates are already in +flight between sensor and actuator, since every one of them is committed before the +correction lands. + ## The RE queue is computed, not argued about ``` @@ -472,7 +554,7 @@ event receiver and silently steals the first one's callbacks. pip install -e '.[dev]' && pytest ``` -149 device-free tests. The ones that matter most try to make the layer lie: +283 device-free tests. The ones that matter most try to make the layer lie: - claim a step is automated when its command is undecoded; claim a decoded command is runnable while its siblings are not; claim a federated leg runs when no run card was ever diff --git a/autonomous_lab/__init__.py b/autonomous_lab/__init__.py index f6a45a9..0451de2 100644 --- a/autonomous_lab/__init__.py +++ b/autonomous_lab/__init__.py @@ -35,9 +35,65 @@ lineage which cell a read came from -- and whether pooling already destroyed the answer, which is a different question from whether the plate was tracked. intelligence the tacit expert judgment and the benchmarks a robot must meet first. + +Four further layers cover what a workcell needs once more than one instrument is involved +and the lab has to keep running: + + coverage vision and QC gates composed. For every failure that destroys material, is + there ANYTHING that would catch it? Where a camera is physically incapable, + an assay is the only option left, so an invisible failure with no gate is + uncovered and no CV budget ever changes that. + durability what an instrument is currently entitled to be trusted with, and whether a + planned campaign crosses a service boundary mid-run. Not a failure + predictor -- that needs reliability data this package does not have. + teaching an expert demonstrating an operation, and a machine measured against that + demonstration. A demonstration is data, not authority: one performance is a + value with no tolerance, and saying so is the point. + feedback whether a control loop can actually close. A sensor downstream of its own + actuator is a post-mortem wearing the costume of a control loop. + +`Envelope` is defined in both `teaching` (what an expert demonstrated) and `feedback` (what +a loop steers toward). They are related and not the same, so neither is re-exported here -- +import from the module that means the one you want. """ +from .coverage import ( + Ceiling, + Cover, + CoverageReport, + CoverageRow, + Demand, + SotaLift, + coverage_report, + mandatory_gates, + sota_lift, + unmet_demands, +) +from .durability import ( + Accrued, + BoundaryReport, + Campaign, + Entitlement, + InstrumentHealth, + Interval, + IntervalKind, + ServiceRecord, + crosses_boundary, + entitlement_summary, + untrusted_instruments, +) from .executor import Executor, Handoff, RunReport, StepResult +from .feedback import ( + Closable, + Closure, + Controller, + Correction, + FeedbackReport, + Loop, + can_close, + feedback_report, + in_flight_exposure, +) from .intelligence import ( Benchmark, BenchmarkStatus, @@ -66,6 +122,20 @@ from .model import Artifact, Protocol, Role, Step, Tier, Transform, Verdict, ZeroDecodeOp from .provenance import Attestation, CustodyGap, Event, RunRecord, provenance_report from .qc import Basis, Criterion, Decision, Gate, Readiness, evaluate, gate_report +from .teaching import ( + Attainment, + Demonstration, + MachineObservation, + NextDemonstration, + TransferReport, + TransferRow, + attainment, + demonstration_queue, + envelope_for, + taught, + transfer_report, + untaught_operations, +) from .recovery import Detection, FailureMode, Latency, Severity, recovery_report from .registry import FEDERATED, FederatedSpec, InstrumentSpec, declared, registry, spec from .throughput import Duration, TimeBasis, estimate @@ -91,27 +161,45 @@ def loop_closure_for(protocol, workcell=None): __all__ = [ + "Accrued", "Artifact", - "Cohort", - "MISASSIGNMENT", + "Attainment", "Attestation", "Basis", "Benchmark", "BenchmarkStatus", + "BoundaryReport", + "Campaign", + "Ceiling", + "Closable", + "Closure", + "Cohort", + "Controller", + "Correction", + "Cover", + "CoverageReport", + "CoverageRow", "Criterion", "CustodyGap", "Decision", + "Demand", + "Demonstration", "Detection", "Duration", + "Entitlement", "Event", "Executor", "FEDERATED", "FailureMode", "FederatedSpec", + "FeedbackReport", "Gate", "Handoff", "InstrumentConfig", + "InstrumentHealth", "InstrumentSpec", + "Interval", + "IntervalKind", "Judgment", "Latency", "Ledger", @@ -119,22 +207,30 @@ def loop_closure_for(protocol, workcell=None): "LineageEdge", "LineageGraph", "LineageReport", - "Misassignment", + "Loop", "LoopClosure", + "MISASSIGNMENT", + "MachineObservation", + "Misassignment", + "NextDemonstration", "Observable", "Protocol", "Readiness", "Role", "RunRecord", - "Separability", "RunReport", + "Separability", + "ServiceRecord", "Severity", + "SotaLift", "Step", "StepResult", "StepVerdict", "Tier", "TimeBasis", "Traceability", + "TransferReport", + "TransferRow", "Transform", "UndeclaredTransform", "Unlock", @@ -144,22 +240,38 @@ def loop_closure_for(protocol, workcell=None): "VisualCheck", "Workcell", "ZeroDecodeOp", + "attainment", "build_ledger", "build_lineage", + "can_close", "cost_step", + "coverage_report", + "crosses_boundary", "declared", + "demonstration_queue", + "entitlement_summary", + "envelope_for", "estimate", "evaluate", + "feedback_report", "gate_report", + "in_flight_exposure", "knowledge_summary", "lineage_report", "loop_closure", "loop_closure_for", + "mandatory_gates", "provenance_report", "rank_unlocks", "recovery_report", "registry", + "sota_lift", "spec", + "taught", + "transfer_report", "trusted_for", "undeclared_transforms", + "unmet_demands", + "untaught_operations", + "untrusted_instruments", ] diff --git a/autonomous_lab/cli.py b/autonomous_lab/cli.py index 16c258a..4a0d2e7 100644 --- a/autonomous_lab/cli.py +++ b/autonomous_lab/cli.py @@ -31,7 +31,11 @@ loop_closure, untrusted_ops, ) +from .coverage import coverage_report, mandatory_gates +from .durability import entitlement_summary, untrusted_instruments +from .feedback import feedback_report from .ledger import build_ledger, rank_unlocks +from .teaching import demonstration_queue, teaching_summary, transfer_report from .lineage import MISASSIGNMENT, Separability, lineage_report from .model import Verdict from .provenance import provenance_report @@ -380,6 +384,153 @@ def _lineage(args) -> int: return 0 if report.verdict.ok else 1 +def _coverage(args) -> int: + """Vision and QC gates composed: would anything catch each failure?""" + wc = _workcell(args) + p = protocols.get(args.protocol) + ledger = build_ledger(p, wc) + gates = gate_report(p.name, ledger) + cap = VisionCapability.none() + report = coverage_report(p, ledger, gates, recovery_report(p, gates, cap), cap) + c = report.counts() + + print(f"detection coverage for {p.name}\n") + print(f" failure modes {c['total']}") + print(f" covered {c['covered']} (vision {c['vision']}, gate {c['gate']}, both {c['both']})") + print(f" uncovered {c['uncovered']}, of which {c['uncovered_destructive']} destroy material") + print(f" closable as it stands {c['fixable']} (build the detector, or make the gate fire)") + print(f" needs a new instrument {c['structural']}\n") + + # FIXABLE covers two different jobs -- a detector nobody built, and a gate whose input + # nobody made arrive. Printing them under one heading would tell a reader to point a + # camera at a condition no camera resolves, which is the exact error this module exists + # to prevent, committed by its own report. + fixable = report.fixable() + by_vision = [r for r in fixable if r.observable.reachable_by_vision] + by_repair = [r for r in fixable if not r.observable.reachable_by_vision] + + if by_vision: + print(" A CAMERA WOULD CATCH THESE -- the check is buildable, nobody built it:") + for r in by_vision: + print(f" {r.failure.name:34} ({r.observable.value})") + if r.vision_reason: + print(f" {r.vision_reason[:96]}") + print() + + if by_repair: + print(" INVISIBLE, BUT A DECLARED GATE ALREADY READS THEM -- make the gate fire:") + for r in by_repair: + print(f" {r.failure.name:34} ({r.observable.value})") + if r.gate_reason: + print(f" {r.gate_reason[:96]}") + print() + + structural = report.structural() + if structural: + print(" NO CAMERA REACHES THESE AT ANY CAPABILITY -- they need an assay:") + for r in structural: + print(f" {r.failure.name:34} ({r.observable.value})") + print() + + demands = mandatory_gates(gates) + if demands: + print(f" {len(demands)} MANDATORY GATE(S) -- invisible and destructive, so a gate is the only option:") + for d in demands: + print(f" {d.failure.name}") + print(f" {d.reason[:100]}") + print() + + print( + " Vision and gates are complementary instruments, not alternatives. Where the photons\n" + " are identical either way, no model resolves it and only an assay does." + ) + return 0 if report.complete() else 1 + + +def _durability(args) -> int: + """What each instrument is currently entitled to be trusted with.""" + wc = _workcell(args) + s = entitlement_summary(wc) + print(f"instrument entitlement for workcell '{wc.name}'\n") + for k, v in sorted(s.items()): + print(f" {k:24} {v}") + print() + rows = untrusted_instruments(wc) + if not rows: + print(" Every instrument is currently entitled.") + return 0 + print(f" {len(rows)} instrument(s) not currently entitled:\n") + for u in rows: + print(f" {getattr(u, 'instrument', '?')} [{getattr(getattr(u, 'standing', None), 'value', '?')}]") + print(f" {getattr(u, 'reason', '')[:110]}") + restore = getattr(u, "restores_it", "") + if restore: + print(f" restores it: {restore[:110]}") + print() + print( + " An instrument with no service history is not a healthy instrument, it is an\n" + " unmeasured one -- the same convention this package applies to benchmarks." + ) + return 1 + + +def _teaching(args) -> int: + """What an expert has demonstrated, and what the machine attains against it.""" + p = protocols.get(args.protocol) + ops = sorted({s.op for s in p.steps}) + report = transfer_report(ops) + s = teaching_summary() + print(f"expert transfer for {p.name}\n") + for k, v in sorted(s.items()): + print(f" {k:32} {v}") + print() + for row in report.rows: + print(f" {row.operation:34} {row.parity.attainment.value}") + print(f" expert n={row.parity.expert_n}, machine n={row.parity.machine_n}") + if row.parity.reason: + print(f" {row.parity.reason[:104]}") + queue = demonstration_queue([p]) + if queue: + print("\n WHAT TO DEMONSTRATE NEXT (ranked by steps unblocked):") + for n in queue[:6]: + ops = ", ".join(n.operations[:3]) + print(f" {n.instrument:16} {n.demonstrations_needed} demo(s) -> unblocks {n.steps_blocked} step(s)") + print(f" {ops}") + print( + "\n A demonstration is data, not authority. One performance is a value with no\n" + " tolerance, and a tolerance invented around it would be a fabricated number." + ) + return 0 + + +def _feedback(args) -> int: + """Can a control loop on this protocol actually close?""" + wc = _workcell(args) + p = protocols.get(args.protocol) + report = feedback_report(p, build_ledger(p, wc)) + print(f"control loops for {p.name}\n") + if not report.rows: + print(" No control loop is declared for this protocol.") + print(" A protocol with no loop is open-loop by construction, however automated it is.") + return 1 + for row in report.rows: + verdict = getattr(getattr(row, "closable", None), "value", "?") + print(f" {getattr(getattr(row, 'loop', None), 'name', '?'):28} {verdict.upper()}") + reason = getattr(row, "reason", "") + if reason: + print(f" {reason[:110]}") + exposure = getattr(row, "in_flight", None) + if exposure is not None: + print(f" committed before a correction lands: {exposure}") + print() + print( + " A sensor downstream of its own actuator is a post-mortem wearing the costume of a\n" + " control loop. This module does not model gain or settling -- that needs a plant\n" + " model and measured response data nothing here has." + ) + return 0 if all(getattr(getattr(r, "closable", None), "value", "") == "closes" for r in report.rows) else 1 + + def _knowledge(args) -> int: """The tacit layer: expert judgments and the benchmarks a robot must meet.""" s = knowledge_summary() @@ -523,6 +674,25 @@ def common(sp): ln.add_argument("--unit", default="cell", help="what the science needs to distinguish") ln.set_defaults(func=_lineage) + cvg = sub.add_parser("coverage", help="would anything catch each failure? vision and gates composed") + common(cvg) + cvg.add_argument("protocol", nargs="?", default="single_cell_genomics") + cvg.set_defaults(func=_coverage) + + dur = sub.add_parser("durability", help="what each instrument is entitled to be trusted with") + common(dur) + dur.set_defaults(func=_durability) + + tch = sub.add_parser("teaching", help="what an expert demonstrated, and what the machine attains") + common(tch) + tch.add_argument("protocol", nargs="?", default="single_cell_genomics") + tch.set_defaults(func=_teaching) + + fbk = sub.add_parser("feedback", help="can a control loop on this protocol actually close?") + common(fbk) + fbk.add_argument("protocol", nargs="?", default="single_cell_genomics") + fbk.set_defaults(func=_feedback) + kn = sub.add_parser("knowledge", help="encoded expert judgment and robot benchmarks") common(kn) kn.set_defaults(func=_knowledge) diff --git a/autonomous_lab/coverage.py b/autonomous_lab/coverage.py new file mode 100644 index 0000000..0cd157f --- /dev/null +++ b/autonomous_lab/coverage.py @@ -0,0 +1,749 @@ +"""Where vision and QC gates meet: for every failure that destroys material, what catches it. + +Three layers in this package each hold a third of one answer and none of them can state it. +`vision` says what a camera can and cannot ever see, and INVISIBLE there is a claim about +physics rather than about model quality. `qc` says which gates exist and whether their +inputs arrive at all. `recovery` says which failures exist, what each one costs, and how +late its declared path would find it. A lab asks none of those questions on its own. It asks +one question, and nobody composes the three layers well enough to answer it: FOR EVERY +FAILURE THAT DESTROYS MATERIAL, IS THERE ANYTHING AT ALL THAT WOULD CATCH IT? + +Composing them is not bookkeeping, because the two instruments are complementary rather than +alternative. Where a camera is physically incapable a gate is the ONLY remaining option, so +an INVISIBLE failure with no gate is uncovered and no CV budget ever changes that. In the +other direction, a failure a camera catches cheaply does not need an assay, and a lab that +buys one has spent wet-lab money on a problem a lens solves. Neither layer can tell those +two apart on its own. Telling them apart is what this module is for. + +The composition also finds coverage that neither layer reports. `recovery` resolves each +failure's DECLARED path and nothing else, so bead_pellet_aspirated -- which declares a gate +-- comes back silent in that report whether or not a camera would see the pellet. A visual +check for exactly that condition is declared in `vision`. Asking vision directly instead of +through the declared path is what turns "the plan names a gate and the gate is dead" into "a +camera would hold this one, and here is precisely what the camera still needs". + +THE RULE THIS MODULE EXISTS TO ENFORCE. A vision check that exists and is not deployable is +not coverage. A gate that exists and is UNSATISFIABLE is not coverage. A gate that is +evaluable and rests on the wrong assay for the sample is not coverage either, and that third +one is the quietest of the three: a number arrives, the gate passes, and the library was +never there. All three are the vacuous pass in a new costume. A plan that counts unbuilt +detectors and unevaluable gates reports a lab as covered while its material is being +destroyed silently -- and reports it in the exact language a reviewer would accept. + +COVERED here means covered by something that works with nobody in the room, and only two +things qualify: a deployed and validated visual check, and an evaluable gate reading an +appropriate assay. An operator noticing is not coverage in a package about unattended +operation; it is the condition this whole package exists to remove. Data analysis after +sequencing is not coverage of a sample either, because the flow cell is already spent by the +time it speaks. Both are real all the same, so every row carries the path `recovery` +resolved, and nothing here ever claims that nothing anywhere would notice when a sibling +layer says something would. The claim is narrower and worth more: nothing MACHINE-CHECKABLE +stands between this failure and the material. + +`sota_lift` is the buying decision. Given a better vision capability it reports which +failures move from uncovered to covered -- and, more prominently, which do not move at ANY +capability, because they are INVISIBLE. The second list is the point of the function. +Upgrading a model has a ceiling, the ceiling is physics rather than engineering, and a lab +reading only the first list will keep buying model upgrades against failures no model +resolves. + +`mandatory_gates` states the same fact from the other side, as a demand rather than a gap. +For an INVISIBLE failure that destroys material a gate is not one option among several; it +is the only one. Where that gate does not exist the report says a gate MUST exist, which is +a different sentence from "coverage is incomplete" and implies a different budget. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import Dict, List, Optional, Tuple + +from .ledger import build_ledger +from .qc import MEASUREMENTS, GateReport, Measurement, gate_report +from .recovery import ( + FAILURE_MODES, + Detection, + EffectiveDetection, + FailureMode, + Latency, + RecoveryReport, + Severity, + recovery_report, +) +from .vision import Observable, VisionCapability, VisionRequirement, check_catching + + +def _destroys_material(failure: FailureMode) -> bool: + """Whether this failure destroys material, asked of the layer that draws that line. + + `recovery.EffectiveDetection.destructive_and_silent` already decides which severities count + as destruction. Restating the severity set here would be a second definition of one line, + and two definitions of one line eventually disagree -- invisibly, because both keep + returning a plausible boolean. So the question is put to recovery in the only form recovery + answers it: hold the latency at NEVER, which satisfies the silent half of its conjunction, + and what the property reports back is the severity half on its own. + """ + probe = EffectiveDetection( + failure, Detection.NONE, Latency.NEVER, "a severity probe, not a finding about this lab" + ) + return probe.destructive_and_silent + + +class Cover(str, Enum): + """How a failure is actually caught in a given workcell. + + Only two instruments can put a member other than NEITHER on a row, and the restriction is + deliberate. A visual check and a QC gate are the two things that keep working when the lab + is running unattended, which is the only condition this package cares about. An operator's + eye and a post-run analysis are real and are not this: one requires the human the lab is + trying to do without, the other speaks after the flow cell is already spent. + + BOTH is not redundancy for its own sake. It is the only state where one instrument can go + down -- a fouled lens, a reader out for service -- and the failure is still caught, which + is a different property from being covered today and worth being able to see. + """ + + VISION = "vision" # a declared visual check is deployed and validated here + GATE = "gate" # a declared gate is evaluable and reads an appropriate assay + BOTH = "both" # either alone would hold it + NEITHER = "neither" # no camera and no gate; the row says which kind of nothing + + @property + def covered(self) -> bool: + """True where something machine-checkable stands between the failure and the material.""" + return self is not Cover.NEITHER + + @property + def closed_by(self) -> str: + """What holds this failure, named -- or, for NEITHER, that nothing does. + + Prose rather than a structure because the caller printing a coverage table needs a + phrase, and a phrase that says "covered" without naming the instrument is how a table + survives an audit it should not survive. + """ + if self is Cover.VISION: + return "a deployed, validated visual check" + if self is Cover.GATE: + return "an evaluable gate reading an assay appropriate to the sample" + if self is Cover.BOTH: + return "a deployed visual check and an evaluable gate; either alone would hold it" + return "nothing; the row states whether a detector or a different instrument would close it" + + +class Uncovered(str, Enum): + """Why a failure is uncovered. Two cases that produce the same NEITHER and are not alike. + + This distinction is the reason NEITHER is not the end of the analysis. Both cases print as + an empty coverage cell and both feel like the same problem to whoever reads the report, and + they have nothing in common except that appearance. + + UNCOVERED_FIXABLE is a detector nobody has built, or a gate whose input nobody has made + arrive. The work is known, it is schedulable, and a capability upgrade or an instrument + repair closes it. + + UNCOVERED_STRUCTURAL is a failure that is physically invisible AND that no declared assay + reads. No camera resolves it at any model quality and no gate in the protocol looks at the + quantity it changes, so the only fixes are a sensor this lab does not have or a protocol + that never creates the condition. Spending a CV budget against one of these buys nothing, + and a report that cannot tell the two apart will recommend exactly that. + """ + + FIXABLE = "uncovered_fixable" + STRUCTURAL = "uncovered_structural" + + @property + def needs_a_new_instrument(self) -> bool: + """True only for STRUCTURAL: the line between schedulable work and a different sensor.""" + return self is Uncovered.STRUCTURAL + + +@dataclass(frozen=True) +class CoverageRow: + """One failure, both instruments resolved against this workcell, and what closes the gap. + + Severity, Observable, the Cover, and the uncovered kind are all METHODS rather than fields. + Every one of them is derivable from what is already stored, and a derived value kept beside + the values it derives from eventually disagrees with them, with nothing to reveal which one + is wrong. This matters more here than elsewhere in the package: a stored `cover` field would + be an author-settable claim of coverage, which is precisely the thing the module exists to + refuse. + + `effective` is what `recovery` computed for this failure and is never recomputed here. It is + carried so that a NEITHER row can say what else, if anything, would surface the failure -- + the STAR raising USBError is real coverage of star_resource_busy and no camera is needed for + it, and a report that demanded one would be noise. + """ + + failure: FailureMode + effective: EffectiveDetection + vision_check: Optional[str] # the declared check that watches this failure, if any + vision_deployable: bool # exists AND every requirement is met in this workcell + vision_reason: str + gate: Optional[str] # the declared gate that would reveal it, if any + gate_evaluable: bool # exists AND its inputs arrive AND the assay suits the sample + gate_reason: str + + @property + def severity(self) -> Severity: + return self.failure.severity + + @property + def observable(self) -> Observable: + return self.failure.observable + + @property + def cover(self) -> Cover: + """The composition, and the only place the two arms are combined. + + Both arms are already the strict form of their question -- exists AND is usable -- so + this is a plain conjunction. It is written out rather than inlined at the call sites + because a second combination of these two booleans somewhere else is how a report starts + counting an unbuilt detector. + """ + if self.vision_deployable and self.gate_evaluable: + return Cover.BOTH + if self.vision_deployable: + return Cover.VISION + if self.gate_evaluable: + return Cover.GATE + return Cover.NEITHER + + @property + def uncovered(self) -> Optional[Uncovered]: + """Which kind of nothing this is, or None when something holds it. + + STRUCTURAL requires both halves: no camera can ever see it, and no gate in this protocol + reads what it changes. A failure that is invisible but that a declared gate reads is not + structural, however dead that gate is today -- the assay exists, and repairing an + instrument is a different budget from commissioning a sensor. + """ + if self.cover.covered: + return None + if self.observable.reachable_by_vision: + return Uncovered.FIXABLE + if self.gate is None: + return Uncovered.STRUCTURAL + return Uncovered.FIXABLE + + @property + def destroys_material(self) -> bool: + return _destroys_material(self.failure) + + @property + def residual(self) -> Optional[Detection]: + """The path `recovery` resolved that is neither a camera nor a gate, where one survives. + + None when something machine-checkable already holds the failure, and None when recovery + found nothing either. Anything else is a real path this module deliberately does not count + as coverage, and reporting it is what keeps an uncovered row from being read as a claim + that the failure is invisible to the entire lab. + """ + if self.cover.covered or self.effective.silent: + return None + return self.effective.detection + + @property + def unwatched(self) -> bool: + """No camera, no gate, and nothing else either. The rows that should be read first.""" + return not self.cover.covered and self.effective.silent + + def closes_it(self) -> str: + """The next action that would put a machine check between this failure and the material. + + One sentence, specific enough to be scheduled. A coverage report whose remedy column reads + "improve coverage" has handed the problem back to the reader with a number attached. + """ + if self.cover.covered: + return f"already held by {self.cover.closed_by}" + + if self.uncovered is Uncovered.STRUCTURAL: + residual = self.residual + if residual is not None: + tail = ( + "; that is after the flow cell is spent, so it prices the loss rather than " + "preventing it" + if self.effective.latency is Latency.AFTER_SEQUENCING + else "" + ) + return ( + f"no camera resolves '{self.failure.name}' and no declared gate reads what it " + f"changes. The only thing standing on it is the declared {residual.value} path, " + f"which finds it {self.effective.latency.value}{tail}" + ) + return ( + f"no camera resolves '{self.failure.name}' and no declared gate reads what it " + f"changes, so nothing schedulable closes it: it needs a sensor this workcell does " + f"not have, or a protocol that never creates the condition" + ) + + options: List[str] = [] + if self.observable.reachable_by_vision: + if self.vision_check is not None: + options.append(f"deploy '{self.vision_check}' -- {self.vision_reason}") + else: + options.append( + "the condition is imageable and no visual check is declared for it; a check has to " + "be written before a camera buys anything" + ) + if self.gate is not None: + options.append(f"make gate '{self.gate}' count -- {self.gate_reason}") + if not options: + # Unreachable while `uncovered` is computed as it is above. Kept because the alternative + # to an explicit refusal is an empty remedy string, which reads as "nothing to do". + return "uncovered, and this module cannot name what would close it" + return "; or ".join(options) + + +@dataclass +class CoverageReport: + """Every failure on a protocol, with both instruments resolved against one workcell.""" + + protocol: str + capability: VisionCapability + rows: List[CoverageRow] = field(default_factory=list) + + def covered(self) -> List[CoverageRow]: + return [r for r in self.rows if r.cover.covered] + + def uncovered(self) -> List[CoverageRow]: + return [r for r in self.rows if not r.cover.covered] + + def fixable(self) -> List[CoverageRow]: + return [r for r in self.rows if r.uncovered is Uncovered.FIXABLE] + + def structural(self) -> List[CoverageRow]: + """Uncovered, and no capability or repair closes it. The re-instrument list.""" + return [r for r in self.rows if r.uncovered is Uncovered.STRUCTURAL] + + def uncovered_destructive(self) -> List[CoverageRow]: + """Destroys material, and no camera or gate stands on it. The headline of the report. + + Ordered worst first, and never filtered by whether some other path exists. A row here + whose residual path is an operator's eye still belongs here: the operator is the thing an + autonomous lab removes, so a run-destroying failure held only by a person is a hole the + moment the lab does what it was built to do. + """ + out = [r for r in self.rows if not r.cover.covered and r.destroys_material] + out.sort(key=lambda r: (not r.unwatched, r.failure.name)) + return out + + def unwatched(self) -> List[CoverageRow]: + """No camera, no gate, and `recovery` found nothing else.""" + return [r for r in self.rows if r.unwatched] + + def complete(self) -> bool: + """True only when every material-destroying failure has a machine check on it. + + `bool(self.rows)` is load-bearing, not defensive. An empty report satisfies the + all-clear by vacuity, and a composition that answered "complete" for a protocol it + resolved no failures on would be the same bug this package chases everywhere else: a + check that passes because it was handed nothing. + """ + return bool(self.rows) and not self.uncovered_destructive() + + def counts(self) -> Dict[str, int]: + return { + "total": len(self.rows), + "covered": len(self.covered()), + "vision": sum(1 for r in self.rows if r.cover is Cover.VISION), + "gate": sum(1 for r in self.rows if r.cover is Cover.GATE), + "both": sum(1 for r in self.rows if r.cover is Cover.BOTH), + "uncovered": len(self.uncovered()), + "uncovered_destructive": len(self.uncovered_destructive()), + "fixable": len(self.fixable()), + "structural": len(self.structural()), + "unwatched": len(self.unwatched()), + } + + +def _vision_arm( + failure: FailureMode, capability: VisionCapability +) -> Tuple[Optional[str], bool, str]: + """(check name, deployable, reason) for the camera side of one failure. + + Resolved through `vision.check_catching` rather than through the failure's declared + detection path, and that is the whole reason this module exists. `recovery` consults vision + only for failures whose PLAN names a camera, so a failure whose plan names a gate is + reported silent even where a declared visual check would see it. bead_pellet_aspirated is + exactly that case, and it is the most destructive routine failure in the protocol. + + Deployability is asked of `VisionCapability.available`, which already refuses an impossible + check. Recomputing that test here would be a second opinion on a question vision owns. + """ + check = check_catching(failure.name) + if check is None: + if failure.observable.reachable_by_vision: + return None, False, "the condition is imageable and no visual check is declared for it" + return None, False, "no camera resolves this condition at any model quality" + if capability.available(check): + return check.name, True, f"'{check.name}' is deployed and validated in this workcell" + if not check.possible: + return check.name, False, f"'{check.name}' watches a condition no camera resolves" + missing = capability.missing_for(check) + return ( + check.name, + False, + f"'{check.name}' is declared but not deployable: missing " + + ", ".join(m.value for m in missing), + ) + + +def _gate_arm( + failure: FailureMode, gates: GateReport, measurements: Dict[str, Measurement] +) -> Tuple[Optional[str], bool, str]: + """(gate name, counts as coverage, reason) for the assay side of one failure. + + Three ways a declared gate fails to be coverage, in increasing order of how quietly it + does so. It may not be on this protocol at all. It may be UNSATISFIABLE, which at least + fails loudly -- no number arrives. Or it may be evaluable and resting on a measurement that + is the wrong assay for the sample, which is the one that spends the flow cell: the + instrument returns a real, precise, reproducible number about something other than the + library, and the gate passes on it. + """ + name = failure.via_gate + if name is None: + return None, False, "no gate on this protocol reads what this failure changes" + row = next((r for r in gates.rows if r.gate.name == name), None) + if row is None: + # The name is reported in the reason and deliberately NOT returned as the gate. A gate + # that is not on this protocol reads nothing here, and returning its name would let the + # uncovered classification treat a dangling reference as an assay that exists. + return ( + None, + False, + f"the plan names '{name}' for this failure, and no such gate is on this protocol", + ) + if not row.evaluable: + return name, False, f"'{name}' is unsatisfiable: {row.reason}" + wrong = [m for gate_name, m in gates.inappropriate(measurements) if gate_name == name] + if wrong: + return ( + name, + False, + f"'{name}' can be evaluated and rests on '{wrong[0].key}', which is the wrong assay " + f"for this sample: {wrong[0].inappropriate_reason}", + ) + return name, True, f"'{name}' is evaluable and reads an assay appropriate to the sample" + + +def coverage_report( + protocol, + ledger, + gates: GateReport, + recovery: RecoveryReport, + vision_capability: Optional[VisionCapability] = None, + measurements: Optional[Dict[str, Measurement]] = None, +) -> CoverageReport: + """Compose the three layers into the answer a lab actually asks for. + + Every input is a report some other module already computed, and nothing here is + re-derived: the failure list comes from `recovery`, gate readiness from `qc`, and + deployability from `vision`. That is why this cannot disagree with any of them. + + It refuses two kinds of incoherent input rather than averaging over them. Reports about + different protocols compose into a number about no lab at all. A recovery report built + against a different vision capability than the one passed here does the same thing more + quietly, because every row still looks well-formed -- so the mismatch is checked against + the one place the two are forced to agree: a failure whose declared path is a camera is + resolved by `recovery` exactly when its check is deployable. + """ + cap = vision_capability or VisionCapability.none() + meas = measurements if measurements is not None else MEASUREMENTS + + built_for = { + "ledger": ledger.protocol.name, + "gate report": gates.protocol, + "recovery report": recovery.protocol, + } + wrong = sorted((label, name) for label, name in built_for.items() if name != protocol.name) + if wrong: + raise ValueError( + f"coverage composes reports about one protocol; '{protocol.name}' was asked for but " + + ", ".join(f"the {label} is about '{name}'" for label, name in wrong) + + ". Composing across protocols returns a number about no lab." + ) + + rows: List[CoverageRow] = [] + for eff in recovery.rows: + failure = eff.failure + check_name, deployable, vision_reason = _vision_arm(failure, cap) + gate_name, evaluable, gate_reason = _gate_arm(failure, gates, meas) + if failure.declared_detection is Detection.VISION: + if (eff.detection is Detection.VISION) != deployable: + raise ValueError( + f"the recovery report and the vision capability disagree about " + f"'{failure.name}': recovery resolved it to '{eff.detection.value}' while " + f"'{cap.name}' says the check is " + f"{'deployable' if deployable else 'not deployable'}. The recovery report was " + f"built against a different capability, and composing the two would report a " + f"workcell nobody has." + ) + rows.append( + CoverageRow( + failure=failure, + effective=eff, + vision_check=check_name, + vision_deployable=deployable, + vision_reason=vision_reason, + gate=gate_name, + gate_evaluable=evaluable, + gate_reason=gate_reason, + ) + ) + return CoverageReport(protocol=protocol.name, capability=cap, rows=rows) + + +# -- what a better model buys, and what it does not ----------------------------- + + +@dataclass(frozen=True) +class Lift: + """One failure a better vision capability actually moves from uncovered to covered.""" + + failure: FailureMode + before: Cover + after: Cover + check: str + gained: Tuple[VisionRequirement, ...] # the requirements the proposal adds that this needed + + @property + def destroys_material(self) -> bool: + return _destroys_material(self.failure) + + +@dataclass(frozen=True) +class Ceiling: + """One failure no vision capability moves, at any price, and the reason it cannot. + + Named for what it is. This is not a gap in a roadmap; it is the roof of the room. + """ + + failure: FailureMode + reason: str + + @property + def destroys_material(self) -> bool: + return _destroys_material(self.failure) + + +@dataclass +class SotaLift: + """What a proposed vision capability buys, and what it provably does not. + + Both lists are required output. A report that showed only `lifted` would be a purchase + justification, and every purchase justification for a better model is correct about the + failures it lists and silent about the ones that make the purchase insufficient. + """ + + protocol: str + current: VisionCapability + proposed: VisionCapability + lifted: List[Lift] = field(default_factory=list) + ceiling: List[Ceiling] = field(default_factory=list) + + @property + def buys_nothing(self) -> bool: + return not self.lifted + + def destructive_ceiling(self) -> List[Ceiling]: + """The ceiling rows that cost material. The sentence a budget meeting needs.""" + return [c for c in self.ceiling if c.destroys_material] + + def counts(self) -> Dict[str, int]: + return { + "lifted": len(self.lifted), + "lifted_destructive": sum(1 for lift in self.lifted if lift.destroys_material), + "ceiling": len(self.ceiling), + "ceiling_destructive": len(self.destructive_ceiling()), + } + + +def sota_lift( + current_capability: VisionCapability, + proposed_capability: VisionCapability, + protocol, + ledger=None, + gates: Optional[GateReport] = None, + measurements: Optional[Dict[str, Measurement]] = None, +) -> SotaLift: + """Deploy a better model, or buy an assay? This is the computation that decides. + + Two coverage reports over one protocol and one gate report, differing only in the vision + capability, because gates do not move when a model does. What changes between them is what + the capability buys. What is uncovered and INVISIBLE in the second is the ceiling, and no + further capability moves it -- so a lab reading only the lift will keep spending against + failures that are not a model problem. + """ + led = ledger if ledger is not None else build_ledger(protocol) + gts = gates if gates is not None else gate_report(protocol.name, led) + + before = coverage_report( + protocol, led, gts, recovery_report(protocol, gts, current_capability), current_capability, + measurements, + ) + after = coverage_report( + protocol, led, gts, recovery_report(protocol, gts, proposed_capability), proposed_capability, + measurements, + ) + prior = {row.failure.name: row for row in before.rows} + + lifted: List[Lift] = [] + ceiling: List[Ceiling] = [] + for row in after.rows: + was = prior.get(row.failure.name) + moved = was is not None and not was.cover.covered and row.cover.covered + # A lift has to be attributable to a deployed check. The gate report is shared between + # the two runs, so nothing but vision can move a row -- and the INVISIBLE test is + # unreachable by construction, since VisionCapability.available already refuses an + # impossible check. Both are checked anyway: a lift reported for an INVISIBLE failure is + # the exact overclaim this function exists to prevent, and a guard that never fires costs + # nothing next to one that fires once against a real purchase. + attributable = ( + row.observable.reachable_by_vision + and row.vision_deployable + and row.vision_check is not None + ) + if moved and attributable: + check = check_catching(row.failure.name) + requires = check.requires if check is not None else () + lifted.append( + Lift( + failure=row.failure, + before=was.cover, + after=row.cover, + check=row.vision_check, + gained=tuple( + r + for r in requires + if r in proposed_capability.satisfied and r not in current_capability.satisfied + ), + ) + ) + continue + if not row.cover.covered and row.observable is Observable.INVISIBLE: + # The residual is named where one survives, so that a row like star_resource_busy -- + # invisible, and loudly raised by the instrument itself -- is not read as a hole + # merely because a camera cannot reach it. The ceiling is a statement about vision, + # not a claim that nothing in the lab notices. + residual = row.residual + standing = ( + f" The declared {residual.value} path still stands on it at " + f"{row.effective.latency.value}." + if residual is not None + else "" + ) + ceiling.append( + Ceiling( + failure=row.failure, + reason=( + f"'{row.failure.name}' is INVISIBLE: the failed run and the clean run produce " + f"identical images, so no vision capability reaches it at any model quality. " + f"{row.gate_reason}.{standing}" + ), + ) + ) + + lifted.sort(key=lambda lift: (not lift.destroys_material, lift.failure.name)) + ceiling.sort(key=lambda c: (not c.destroys_material, c.failure.name)) + return SotaLift( + protocol=protocol.name, + current=current_capability, + proposed=proposed_capability, + lifted=lifted, + ceiling=ceiling, + ) + + +# -- the gates that are not optional -------------------------------------------- + + +@dataclass(frozen=True) +class Demand: + """A gate that must exist, because no camera can substitute for it. + + A demand rather than a gap, and the word is the content. A gap invites prioritization + against other gaps. A demand says that for this failure there is no second option: the + condition is invisible, the failure destroys material, and an assay is the only instrument + left. `met` is False by default in every path that cannot prove otherwise, including the + path where nobody supplied a gate report to check against. + """ + + failure: FailureMode + gate: Optional[str] + met: bool + reason: str + + def statement(self) -> str: + """The demand written out, in the form a planning document has to answer.""" + verb = "is met" if self.met else "is NOT met" + named = f"'{self.gate}'" if self.gate else "a gate that does not exist yet" + return ( + f"a gate must exist for '{self.failure.name}' ({self.failure.description}): the " + f"condition is INVISIBLE, so no camera substitutes, and the failure costs " + f"{self.failure.severity.value}. The demand names {named} and {verb} -- {self.reason}" + ) + + +def mandatory_gates( + gates: Optional[GateReport] = None, + measurements: Optional[Dict[str, Measurement]] = None, +) -> Tuple[Demand, ...]: + """For every INVISIBLE failure that destroys material, the gate that has to exist. + + These are the rows where the vision layer's hard boundary becomes a purchasing + requirement. A camera cannot see a denatured enzyme, a short transfer inside the + instrument's own tolerance, or two clear liquids swapped between identical tubes, so for + each of them the assay is not one option among several. It is the option. + + Called with no gate report, every demand comes back unmet. That is not a placeholder: an + unchecked demand is not a met one, which is the same convention `intelligence.trusted_for` + applies to a missing benchmark and `qc.evaluate` applies to a missing measurement. A + default that read as satisfied would let a caller establish full compliance by supplying + nothing at all. + """ + meas = measurements if measurements is not None else MEASUREMENTS + out: List[Demand] = [] + for failure in FAILURE_MODES: + if failure.observable is not Observable.INVISIBLE or not _destroys_material(failure): + continue + if gates is None: + out.append( + Demand( + failure=failure, + gate=failure.via_gate, + met=False, + reason=( + "no gate report was supplied, so this demand stands unresolved; an unchecked " + "demand is not a met one" + ), + ) + ) + continue + name, evaluable, reason = _gate_arm(failure, gates, meas) + out.append(Demand(failure=failure, gate=name, met=evaluable, reason=reason)) + return tuple(out) + + +def unmet_demands( + gates: Optional[GateReport] = None, + measurements: Optional[Dict[str, Measurement]] = None, +) -> Tuple[Demand, ...]: + """The subset of `mandatory_gates` a lab has not answered. Usually all of it.""" + return tuple(d for d in mandatory_gates(gates, measurements) if not d.met) + + +__all__ = [ + "Ceiling", + "Cover", + "CoverageReport", + "CoverageRow", + "Demand", + "Lift", + "SotaLift", + "Uncovered", + "coverage_report", + "mandatory_gates", + "sota_lift", + "unmet_demands", +] diff --git a/autonomous_lab/durability.py b/autonomous_lab/durability.py new file mode 100644 index 0000000..077d9ee --- /dev/null +++ b/autonomous_lab/durability.py @@ -0,0 +1,935 @@ +"""What an instrument is entitled to be trusted with, given what it has done since it was serviced. + +Every other layer in this package reasons about a lab in perfect health. The ledger costs a +step against a decoded command set, `qc` costs a gate against the step that feeds it, and +both quietly assume the instrument on the far end is the instrument the vendor shipped. +Instruments wear. Calibrations expire. Seals, tips, pumps, and lamps have service lives, and +consumables deplete. A schedule built on the perfect-health assumption is a schedule for the +first week. + +The gap this fills is the one a real automation programme hits in month three. The run did +not fail because the code was wrong. It failed because a pump was four hundred hours past +service and nobody was tracking hours, or because the reader was recalibrated in the middle +of a plate and the numbers on either side of that moment are not the same measurement. + +This is the insurance layer, and it is deliberately narrow about what insurance means. It +does NOT predict failures. Predicting them needs reliability data this package does not have +-- population lifetimes, censored failure histories, a duty cycle somebody characterized -- +and inventing an MTBF would be exactly the fabricated number `throughput` already refuses +when it declines to return plates per day over eighteen untimed steps. See `NOT_COMPUTED` at +the bottom of this module, which names each prediction this layer will not make and what it +would take to make it honestly. + +What it tracks instead is ENTITLEMENT: what an instrument is currently entitled to be +trusted with, given when its obligations were last discharged and what it has accumulated +since. Entitlement is a bookkeeping question, not a physics question, so it can be answered +from records a lab already keeps -- and the answer is checkable, which a predicted failure +rate is not. + +Three things make this different from a maintenance calendar. + +UNKNOWN IS NOT OPTIMISTIC. An instrument whose service history nobody kept is not a healthy +instrument; it is an unmeasured one. It reports UNKNOWN, UNKNOWN is not a pass, and the +whole enum is ordered so that the instrument nobody recorded ranks below the one everybody +knows is overdue -- because a lapsed obligation names its own fix and an unrecorded one does +not even establish which state the instrument is in. This is the same convention +`intelligence.trusted_for` applies to an unbenchmarked operation and `qc.evaluate` applies +to an absent measurement. + +AN INTERVAL CARRIES ITS BASIS. A vendor-stated service interval is real evidence and it is +not evidence about THIS duty cycle, which is the identical distinction `qc.Basis` already +draws about thresholds -- so this module reuses that enum rather than mirroring it. A +service interval established on a reference workload does not describe an instrument run +sixteen hours a day on picogram input, and a lab that stores the two indistinguishably has +thrown away the only information that would let it revise one. Note carefully what this does +NOT do: an unvalidated interval still governs entitlement. Basis records how much the +recurrence is worth, and refusing to honor a vendor interval because it is only a vendor +interval would be worse than honoring it. + +THE LOAD-BEARING COMPUTATION IS THE BOUNDARY CROSSING, not the current state. Every +instrument in a workcell can be entitled today and the campaign can still be unrunnable, +because a plate that starts on one side of a calibration boundary and finishes on the other +has an ambiguous provenance that nothing downstream repairs. `crosses_boundary` is the +question a planner has to ask before a campaign rather than after it. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Dict, List, Optional, Tuple + +from .provenance import Attestation +from .qc import Basis + + +class Charge(str, Enum): + """The unit an obligation accrues in. + + The line drawn here is whether the clock runs while the instrument sits idle, and it + splits the three unevenly on purpose. RUNS and HOURS are charged by work: an instrument + doing nothing accumulates neither, so a quiet month genuinely buys budget back. DAYS are + charged by the calendar and nothing stops them. A planner that models only usage-based + wear will believe a mothballed instrument stays fresh, and will discover in the same + week that its calibration expired six months ago and that every result since the last + one it trusted has to be reviewed. + """ + + DAYS = "days" # calendar time since the obligation was last discharged + RUNS = "runs" # plates, cycles, or injections put through it + HOURS = "hours" # powered or actuating hours, as the lab counts them + + @property + def accrues_when_idle(self) -> bool: + return self is Charge.DAYS + + +class IntervalKind(str, Enum): + """What kind of obligation recurs, and whether lapsing it reaches backwards. + + This is the distinction that decides how expensive a lapse is, and it is not the one + people expect. A depleted tip box or an overdue pump service is a forward-looking + problem: it stops the next run and costs nothing already recorded. An expired + CALIBRATION or QUALIFICATION is retrospective. Every number the instrument produced + between the expiry and the discovery was produced by a machine nobody could say was + reading true, and no downstream analysis recovers that -- the data is not wrong, it is + unattributable, which is worse because it looks fine. + """ + + CALIBRATION = "calibration" # the instrument reads true against a reference + QUALIFICATION = "qualification" # it has been shown to perform the task, on this configuration + PREVENTIVE_SERVICE = "preventive_service" # wear parts replaced before they fail + CONSUMABLE = "consumable" # tips, seals, filters, reagent on board + + @property + def invalidates_past_results(self) -> bool: + """True when a lapse casts doubt on data the instrument already produced.""" + return self in (IntervalKind.CALIBRATION, IntervalKind.QUALIFICATION) + + +class Entitlement(str, Enum): + """What an instrument may currently be trusted with. Ordered best to worst. + + UNKNOWN sits at the far end rather than in the middle, and that placement is the whole + argument of this module. An instrument everybody knows is overdue is in a bad state that + names its own fix: perform the service. An instrument whose history nobody kept is in no + known state at all, and restoring it needs both the service AND the record-keeping that + should have been running for the last two years. Ranking UNKNOWN above LAPSED would + reward the lab that stopped writing things down. + + EXPIRING is not a defect and does not mean the instrument is unfit. It means the + instrument is entitled today and the CAMPAIGN AS PLANNED collides with a boundary. The + fix is to the plan or to the schedule, not to the machine. + """ + + ENTITLED = "entitled" # every obligation discharged, with a budget the plan fits inside + EXPIRING = "expiring" # entitled now; the planned work crosses a boundary mid-campaign + LAPSED = "lapsed" # an obligation is past due + UNKNOWN = "unknown" # nobody recorded it, or nobody counted what it has done since + + @property + def trustworthy(self) -> bool: + """True only for ENTITLED. + + UNKNOWN is excluded deliberately and is the reason this property exists rather than a + truthiness check at the call site. The absence of a service record is not evidence of + service, in exactly the way `qc` refuses to read an absent measurement as a pass. + """ + return self is Entitlement.ENTITLED + + @property + def entitled_now(self) -> bool: + """True for work that starts and finishes before any boundary in question. + + Separate from `trustworthy` because EXPIRING must not read as unfit. An instrument two + runs from a calibration boundary is entitled to do one run and is not entitled to do a + campaign of forty, and collapsing those into one answer forces a lab to either idle a + healthy instrument or run the campaign anyway. + """ + return self in (Entitlement.ENTITLED, Entitlement.EXPIRING) + + +# Worse is later. Combining obligations takes the worst, so one unrecorded calibration +# makes the instrument UNKNOWN however complete the rest of its history is. +_ENTITLEMENT_ORDER: Tuple[Entitlement, ...] = ( + Entitlement.ENTITLED, + Entitlement.EXPIRING, + Entitlement.LAPSED, + Entitlement.UNKNOWN, +) + + +@dataclass(frozen=True) +class Interval: + """A recurring obligation on one instrument, and where its recurrence came from. + + An interval may recur on more than one unit at once, and real ones usually do -- a + calibration valid for a year or five hundred plates, whichever arrives first. Every + declared unit is checked independently and the first to run out governs, because an + interval that recurs on days and runs is two obligations sharing a name. + + `basis` is `qc.Basis`, reused rather than restated. A vendor service interval is VENDOR: + real evidence, and not evidence about this duty cycle. `validated` is therefore False for + every interval a lab has not confirmed against its own workload, which is nearly all of + them, and that is a thing to know before treating one as a specification. + + `restores` is prose and deliberately concrete, in the same discipline + `provenance.CustodyGap.closes_it` applies: "improve maintenance" is not an action, "run + the two-point absorbance calibration and record both readings" is. + """ + + name: str + instrument: str # registry key or federated key + kind: IntervalKind + restores: str # what discharging it restores, in bench terms + basis: Basis + every_days: Optional[float] = None + every_runs: Optional[float] = None + every_hours: Optional[float] = None + rationale: str = "" + + def __post_init__(self) -> None: + declared = { + Charge.DAYS: self.every_days, + Charge.RUNS: self.every_runs, + Charge.HOURS: self.every_hours, + } + if all(v is None for v in declared.values()): + # An obligation that recurs on nothing can never come due, so every instrument + # carrying one would report entitled forever. That is the vacuous pass this package + # hunts, arriving through a data table instead of through a function. + raise ValueError( + f"interval '{self.name}' recurs on no unit; an obligation that never comes due is " + "not an obligation, it is a note" + ) + for charge, value in declared.items(): + if value is not None and value <= 0: + raise ValueError( + f"interval '{self.name}' recurs every {value} {charge.value}, which is already past due " + "the moment it is discharged" + ) + + @property + def validated(self) -> bool: + """True only for a recurrence this lab established against its own duty cycle.""" + return self.basis.validated + + def charges(self) -> Tuple[Charge, ...]: + """The units this obligation accrues in. Never empty; __post_init__ refuses that.""" + out: List[Charge] = [] + for charge, value in ( + (Charge.DAYS, self.every_days), + (Charge.RUNS, self.every_runs), + (Charge.HOURS, self.every_hours), + ): + if value is not None: + out.append(charge) + return tuple(out) + + def every(self, charge: Charge) -> Optional[float]: + if charge is Charge.DAYS: + return self.every_days + if charge is Charge.RUNS: + return self.every_runs + return self.every_hours + + def describe(self) -> str: + parts = [f"every {self.every(c)} {c.value}" for c in self.charges()] + return f"{self.name}: {' or '.join(parts)}, whichever arrives first" + + +@dataclass(frozen=True) +class Accrued: + """What an instrument has done since an obligation was last discharged. + + Every field is Optional and None means UNCOUNTED, never zero. The difference is the + whole point: zero runs since the last calibration is a measurement, and a lab that never + counted runs has not made it. Defaulting an uncounted dimension to zero would restore + entitlement to an instrument nobody has been watching, which is the failure this module + exists to prevent. + + Structurally identical to `Campaign` and kept separate anyway, because one is a count of + the past and the other is a claim about the future, and a function that accepts either + where it meant one will eventually be handed the wrong one. + """ + + days: Optional[float] = None + runs: Optional[float] = None + hours: Optional[float] = None + + def charged(self, charge: Charge) -> Optional[float]: + if charge is Charge.DAYS: + return self.days + if charge is Charge.RUNS: + return self.runs + return self.hours + + +@dataclass(frozen=True) +class ServiceRecord: + """One obligation, discharged: what was done, when, and on whose word. + + `attestation` is `provenance.Attestation`, reused for the same reason it exists there. A + maintenance system that marks a service complete because it was scheduled has recorded + intent, not evidence, and an ASSERTED record does not discharge an obligation here -- + the instrument reports UNKNOWN rather than entitled. WITNESSED is the common and + legitimate case: a technician signed for it, which is unverifiable and real. + + `performed` is prose, read back and never parsed. Nothing in this module does date + arithmetic, because a lab that keeps its own calendar keeps it in a format this package + should not be guessing at. The elapsed quantities arrive in `since`, counted by the + caller, so this layer computes over numbers somebody actually measured. + """ + + interval: str # Interval.name + performed: str # when, as the lab recorded it + by: str # who or what discharged it + attestation: Attestation + since: Accrued # what has accumulated on this instrument since, as counted by the caller + evidence: str = "" + + +@dataclass(frozen=True) +class Campaign: + """Planned work, in the units obligations are charged in. + + Days are declared separately from runs on purpose. A campaign that occupies the lab for + ninety days charges ninety days against the calibration of every instrument in it, + including the ones it never touches, and a plan that counts only its own plates will + miss the boundary that catches it. + """ + + runs: float = 0.0 + hours: float = 0.0 + days: float = 0.0 + + def charged(self, charge: Charge) -> float: + if charge is Charge.DAYS: + return self.days + if charge is Charge.RUNS: + return self.runs + return self.hours + + @property + def empty(self) -> bool: + return not (self.runs or self.hours or self.days) + + +@dataclass(frozen=True) +class Remaining: + """How much of one obligation's budget is left, per unit it is charged in. + + `uncounted` is the field that keeps this honest, and it is why the class exists rather + than a float. An interval charged in runs against a lab that never counted runs has no + remaining budget -- not a large one, none that can be stated -- and a report that + returned a number there would be a number over an input nobody measured. + + Everything derivable is a method. A stored `exhausted` flag beside the budget it derives + from eventually disagrees with it, and nothing reveals which one is wrong. + """ + + interval: str + left: Tuple[Tuple[Charge, float], ...] = () + uncounted: Tuple[Charge, ...] = () + recorded: bool = False + + @property + def computable(self) -> bool: + return self.recorded and not self.uncounted + + def value(self, charge: Charge) -> Optional[float]: + for c, v in self.left: + if c is charge: + return v + return None + + def exhausted(self) -> Tuple[Charge, ...]: + """Units whose budget is spent. A budget of exactly zero is spent, not nearly spent.""" + return tuple(c for c, v in self.left if v <= 0) + + def crossed_by(self, campaign: Campaign) -> Tuple[Charge, ...]: + """Units where the planned work consumes more than is left. + + Excludes units already exhausted. Those are a lapse, which is a different finding with + a different fix, and folding them in here would let a report about a plan absorb a + fact about the present. + """ + out: List[Charge] = [] + for c, v in self.left: + if v > 0 and campaign.charged(c) > v: + out.append(c) + return tuple(out) + + +@dataclass(frozen=True) +class Standing: + """One obligation on one instrument, resolved against what the caller actually counted. + + `restores` is the concrete next action and it differs by why the standing is bad: a + lapse is restored by performing the obligation, an unrecorded one by performing it AND + recording it, and a crossing by discharging the obligation before the campaign starts + rather than during it. + """ + + instrument: str + interval: Interval + entitlement: Entitlement + remaining: Remaining + reason: str + restores: str + crossing: Tuple[Charge, ...] = () + + @property + def trustworthy(self) -> bool: + return self.entitlement.trustworthy + + +def _remaining(interval: Interval, record: Optional[ServiceRecord]) -> Remaining: + """Budget left on one obligation. Uncounted units are reported, never defaulted.""" + if record is None: + return Remaining(interval=interval.name, uncounted=interval.charges(), recorded=False) + left: List[Tuple[Charge, float]] = [] + uncounted: List[Charge] = [] + for charge in interval.charges(): + counted = record.since.charged(charge) + if counted is None: + uncounted.append(charge) + continue + every = interval.every(charge) + left.append((charge, float(every) - float(counted))) + return Remaining( + interval=interval.name, + left=tuple(left), + uncounted=tuple(uncounted), + recorded=True, + ) + + +def standing( + health: "InstrumentHealth", + interval: Interval, + campaign: Optional[Campaign] = None, +) -> Standing: + """Resolve one obligation against the records this lab kept and the work it plans. + + The order of the checks is the order of severity and it matters. An obligation with no + record is UNKNOWN before anything else is considered, because there is no baseline to + subtract from; an obligation whose record is only ASSERTED is UNKNOWN for the same + reason `provenance` refuses to call a command log a record of events. + """ + record = health.record_for(interval.name) + remaining = _remaining(interval, record) + + if record is None: + return Standing( + instrument=health.instrument, + interval=interval, + entitlement=Entitlement.UNKNOWN, + remaining=remaining, + reason=( + f"no service record exists for '{interval.name}'; an instrument whose history " + "nobody kept is unmeasured, not healthy" + ), + restores=( + f"{interval.restores}, and record what was done, by whom, and the reading that " + "confirms it. An obligation with no record cannot be discharged retroactively" + ), + ) + + if not record.attestation.is_evidence: + return Standing( + instrument=health.instrument, + interval=interval, + entitlement=Entitlement.UNKNOWN, + remaining=remaining, + reason=( + f"'{interval.name}' is recorded as {record.attestation.value}: software marked it " + "done and nothing read back. That is intent, not evidence that it happened" + ), + restores=( + "have the person or instrument that performed it attest to it, so the record is " + f"witnessed or confirmed rather than asserted; otherwise {interval.restores}" + ), + ) + + if remaining.uncounted: + units = ", ".join(c.value for c in remaining.uncounted) + return Standing( + instrument=health.instrument, + interval=interval, + entitlement=Entitlement.UNKNOWN, + remaining=remaining, + reason=( + f"'{interval.name}' is charged in {units}, and nobody counted {units} since it was " + f"last discharged on {record.performed}. There is no budget to state" + ), + restores=( + f"count {units} against this instrument from now on; until then the interval is a " + "policy nobody is measuring against" + ), + ) + + spent = remaining.exhausted() + if spent: + units = ", ".join(f"{c.value} ({remaining.value(c)} left)" for c in spent) + past = ( + " Every result it produced since is unattributable rather than wrong, which is the " + "harder problem." + if interval.kind.invalidates_past_results + else "" + ) + return Standing( + instrument=health.instrument, + interval=interval, + entitlement=Entitlement.LAPSED, + remaining=remaining, + reason=f"'{interval.name}' is past due on {units}.{past}", + restores=interval.restores, + ) + + if campaign is not None: + crossed = remaining.crossed_by(campaign) + if crossed: + units = ", ".join( + f"{campaign.charged(c)} {c.value} planned against {remaining.value(c)} remaining" + for c in crossed + ) + return Standing( + instrument=health.instrument, + interval=interval, + entitlement=Entitlement.EXPIRING, + remaining=remaining, + reason=( + f"the campaign crosses '{interval.name}' mid-run: {units}. The instrument is " + "entitled today and will not be by the end" + ), + restores=( + f"discharge '{interval.name}' before the campaign starts, or split the campaign " + "at the boundary so no plate spans it" + ), + crossing=crossed, + ) + + budget = ", ".join(f"{remaining.value(c)} {c.value}" for c in interval.charges()) + return Standing( + instrument=health.instrument, + interval=interval, + entitlement=Entitlement.ENTITLED, + remaining=remaining, + reason=f"'{interval.name}' discharged on {record.performed}; {budget} remaining", + restores="", + ) + + +@dataclass(frozen=True) +class InstrumentHealth: + """One instrument's obligations and the records that discharge them. + + Carries no entitlement field, and that absence is enforced by a test. Entitlement is + computed from the records and the plan every time it is asked for, in the same way the + ledger computes verdicts rather than storing them: a stored standing beside the records + it derives from eventually disagrees with them, and an author-settable "healthy" flag is + the single easiest way to silence this whole report. + + An instrument with no declared obligations is UNKNOWN rather than entitled. Declaring + none is not the same as having none, and treating an empty obligation list as a clean + bill of health is exactly the vacuous pass a gate evaluated against an empty measurement + dict gives. + """ + + instrument: str + obligations: Tuple[Interval, ...] = () + records: Tuple[ServiceRecord, ...] = () + note: str = "" + + @classmethod + def unrecorded(cls, instrument: str) -> "InstrumentHealth": + """The honest zero state: an instrument nobody has written anything down about.""" + return cls( + instrument=instrument, + note=( + "no service interval is declared and no service record exists for this instrument " + "in this repo" + ), + ) + + def record_for(self, interval_name: str) -> Optional[ServiceRecord]: + """The most recent record for an obligation, taking the last declared as most recent. + + Order is the caller's, not a sort. Sorting would need dates parsed out of `performed`, + and this module does no date arithmetic on purpose. + """ + found: Optional[ServiceRecord] = None + for rec in self.records: + if rec.interval == interval_name: + found = rec + return found + + def standings(self, campaign: Optional[Campaign] = None) -> Tuple[Standing, ...]: + return tuple(standing(self, i, campaign) for i in self.obligations) + + def entitlement(self, campaign: Optional[Campaign] = None) -> Entitlement: + """The worst standing among this instrument's obligations. + + Returns UNKNOWN for an instrument with no obligations declared. That is the direction + that fails safe: a new instrument arrives untrusted and has to earn its way out, + rather than arriving perfect because nobody has written its maintenance plan yet. + """ + if not self.obligations: + return Entitlement.UNKNOWN + worst = Entitlement.ENTITLED + for row in self.standings(campaign): + if _ENTITLEMENT_ORDER.index(row.entitlement) > _ENTITLEMENT_ORDER.index(worst): + worst = row.entitlement + return worst + + def lapsed(self) -> Tuple[Standing, ...]: + return tuple(r for r in self.standings() if r.entitlement is Entitlement.LAPSED) + + def unrecorded_obligations(self) -> Tuple[Standing, ...]: + """Obligations this lab cannot say anything about, for any of the three reasons.""" + return tuple(r for r in self.standings() if r.entitlement is Entitlement.UNKNOWN) + + def unvalidated(self) -> Tuple[Interval, ...]: + """Obligations whose recurrence this lab has not confirmed against its own duty cycle. + + Reported, and deliberately not acted on. A vendor interval still governs entitlement -- + ignoring it because it came from a manual would be worse than honoring it. This lists + the ones whose NUMBER is borrowed, so a lab can put them in a queue to be established + on its own workload, exactly as `qc.Gate.unvalidated` does for thresholds. + """ + return tuple(i for i in self.obligations if not i.validated) + + def invalidating(self) -> Tuple[Standing, ...]: + """Lapses that reach backwards into data the instrument already produced.""" + return tuple(r for r in self.lapsed() if r.interval.kind.invalidates_past_results) + + +# -- the boundary crossing ------------------------------------------------------ + + +@dataclass(frozen=True) +class BoundaryReport: + """Whether a planned campaign can run start to finish on one instrument's entitlement. + + `clear` is the field to read, and it is False for an instrument with no declared + obligations. That case is the one worth stating explicitly: the natural implementation + of "does anything cross" is an `all()` over the obligation list, and an `all()` over an + empty list is True. An instrument nobody wrote a maintenance plan for would then clear + every campaign it was ever checked against. + """ + + instrument: str + campaign: Campaign + rows: Tuple[Standing, ...] + + def crossings(self) -> Tuple[Standing, ...]: + """Obligations the campaign crosses mid-run. The finding this report exists for.""" + return tuple(r for r in self.rows if r.entitlement is Entitlement.EXPIRING) + + def lapsed(self) -> Tuple[Standing, ...]: + return tuple(r for r in self.rows if r.entitlement is Entitlement.LAPSED) + + def uncertain(self) -> Tuple[Standing, ...]: + """Obligations that cannot be shown to be crossed or not crossed. + + Reported alongside real crossings rather than under them. An uncounted budget is not a + small risk, it is an unavailable answer, and a campaign planned over one is planned + over nothing. + """ + return tuple(r for r in self.rows if r.entitlement is Entitlement.UNKNOWN) + + @property + def clear(self) -> bool: + return bool(self.rows) and all(r.trustworthy for r in self.rows) + + def why_not(self) -> str: + """One line naming what stands between this campaign and a clean start.""" + if self.clear: + return f"every obligation on {self.instrument} outlasts the campaign as planned" + if not self.rows: + return ( + f"no service interval is declared for {self.instrument}; an instrument with no " + "maintenance plan has not been shown to need none" + ) + parts: List[str] = [] + for row in self.rows: + if not row.trustworthy: + parts.append(f"{row.interval.name} ({row.entitlement.value})") + return "; ".join(parts) + + +def crosses_boundary( + health: InstrumentHealth, + campaign: Campaign, +) -> BoundaryReport: + """Does the planned campaign cross a service or calibration boundary mid-run? + + This is the insurance question, and it is not answered by checking that every instrument + is entitled today. A plate that starts before a calibration expires and finishes after it + has an ambiguous provenance, and NOTHING downstream can repair that afterwards. + + Tie this to the attestation vocabulary in `provenance`, because the two are easy to + confuse and only one of them is being solved here. A run record can be CONFIRMED at every + step in that module's strongest sense -- the instrument reported completion and the + report was read back -- and still not say which side of the boundary the plate was on, + because the boundary is not an event any instrument reports. Attestation answers WHO SAYS + THE STEP HAPPENED. Entitlement answers WHETHER THE INSTRUMENT WAS ALLOWED TO DO IT. A + record that is unimpeachable about the first and silent about the second leaves the run + exactly as defensible as no record at all, and the hash chain will verify perfectly while + it does. + + A campaign that plans no work is refused rather than reported clear. Asking whether zero + plates cross a boundary is a question about nothing, and returning a clean report to it + would be the emptiest kind of pass this package refuses. + """ + if campaign.empty: + raise ValueError( + "a campaign that plans no runs, hours, or days crosses nothing; state the planned " + "work before asking whether it fits inside the instrument's entitlement" + ) + return BoundaryReport( + instrument=health.instrument, + campaign=campaign, + rows=health.standings(campaign), + ) + + +# -- the workcell view ---------------------------------------------------------- + + +@dataclass(frozen=True) +class Untrusted: + """One instrument that is not currently entitled, and what would restore it. + + `restores` is a tuple because an instrument is usually untrusted for more than one + reason at once, and collapsing them to the worst one hides work. An instrument needing a + calibration AND a pump service is two jobs, and a report that names one of them gets the + second discovered on the morning of the campaign. + """ + + instrument: str + entitlement: Entitlement + reason: str + restores: Tuple[str, ...] + + +def untrusted_instruments( + workcell, + health: Optional[Dict[str, InstrumentHealth]] = None, + campaign: Optional[Campaign] = None, +) -> List[Untrusted]: + """Which instruments in a workcell are not currently entitled, and what would fix each. + + The instrument list is resolved from the workcell rather than from the health dictionary, + which is the whole reason the workcell is a parameter. Iterating the health records would + report on the instruments somebody bothered to write down and silently omit the ones + nobody did -- and those are precisely the instruments this module is about. Federated + instruments are included: running on hardware in another repo says nothing about when the + box was last serviced. + + A missing health record is not a missing row. It resolves to `InstrumentHealth.unrecorded` + and reports UNKNOWN, in the same way `throughput.duration_for` resolves an untimed op to + UNKNOWN rather than to zero. + """ + supplied = health or {} + keys: List[str] = list(workcell.present_keys()) + for key in workcell.federated: + if key not in keys: + keys.append(key) + + out: List[Untrusted] = [] + for key in keys: + h = supplied.get(key) or health_for(key) + verdict = h.entitlement(campaign) + if verdict.trustworthy: + continue + if not h.obligations: + out.append( + Untrusted( + instrument=key, + entitlement=verdict, + reason=( + f"no service interval is declared for '{key}' and no service record exists; " + "an instrument nobody tracked is unmeasured, not healthy" + ), + restores=( + "write down what this instrument is obliged to have done to it and how often, " + "then record the next time each one is discharged", + ), + ) + ) + continue + bad = [r for r in h.standings(campaign) if not r.trustworthy] + out.append( + Untrusted( + instrument=key, + entitlement=verdict, + reason="; ".join(r.reason for r in bad), + restores=tuple(r.restores for r in bad if r.restores), + ) + ) + return out + + +def entitlement_summary( + workcell, + health: Optional[Dict[str, InstrumentHealth]] = None, + campaign: Optional[Campaign] = None, +) -> Dict[str, int]: + """Counts for the report. Every instrument in the workcell is in exactly one bucket.""" + supplied = health or {} + keys: List[str] = list(workcell.present_keys()) + for key in workcell.federated: + if key not in keys: + keys.append(key) + counts = {e.value: 0 for e in Entitlement} + counts["total"] = len(keys) + for key in keys: + h = supplied.get(key) or health_for(key) + counts[h.entitlement(campaign).value] += 1 + return counts + + +# -- what this lab has recorded ------------------------------------------------- +# Nothing, and the empty tables below are the finding rather than an omission. They are +# empty in exactly the way `throughput.DURATIONS` is almost entirely UNKNOWN and +# `vision.VisionCapability.none` satisfies no requirement. +# +# The alternative was to fill them in, and it is worth being explicit about why that would +# be worse than leaving them bare. A plausible-looking "preventive service every 2000 +# hours" written here would be a number nobody in this lab measured, sitting in a table +# other modules read, and within two releases it would be quoted as this lab's service +# interval. That is the precise mechanism by which an estimate becomes a specification, and +# it is the one `throughput` refuses when it declines to publish plates per day. A lab +# filling these in should fill them in from its own instrument manuals and its own +# maintenance history, and should record which of the two each number came from -- that is +# what `Interval.basis` is for. + +OBLIGATIONS: Dict[str, Tuple[Interval, ...]] = {} + +SERVICE_HISTORY: Dict[str, Tuple[ServiceRecord, ...]] = {} + + +def health_for(instrument: str) -> InstrumentHealth: + """This lab's recorded health for an instrument, defaulting to nothing recorded. + + Defaulting rather than raising is the safe direction, and it is the same choice + `throughput.duration_for` makes: a new instrument should make this report less confident, + not crash it. + """ + obligations = OBLIGATIONS.get(instrument, ()) + records = SERVICE_HISTORY.get(instrument, ()) + if not obligations and not records: + return InstrumentHealth.unrecorded(instrument) + return InstrumentHealth(instrument=instrument, obligations=obligations, records=records) + + +# -- what this module will not compute ------------------------------------------ + + +@dataclass(frozen=True) +class Refusal: + """One number this layer declines to return, and what it would take to return it. + + Recorded as data rather than left in prose so the refusal is checkable. A module that + says in its docstring that it does not predict failures and then exposes a + remaining-life helper has documented a discipline it does not have. + """ + + quantity: str + why: str + what_it_would_take: str + + +NOT_COMPUTED: Tuple[Refusal, ...] = ( + Refusal( + quantity="mean time between failures", + why=( + "an MTBF is a population statistic. It is estimated from many units of the same " + "model failing over time, and this lab has one of each instrument and no failure " + "history at all. A figure computed from one box that has not broken yet is not a " + "small-sample estimate, it is arithmetic on nothing" + ), + what_it_would_take=( + "failure and censoring times across a population of the same model under a " + "comparable duty cycle, which is a vendor's or a fleet operator's data and not this " + "lab's" + ), + ), + Refusal( + quantity="failure rate", + why=( + "a rate implies a hazard function, which implies a wear model somebody fitted. " + "Nothing here fits one. Publishing a rate would put a curve nobody measured under " + "every schedule downstream of it, and the schedule would look quantitative" + ), + what_it_would_take=( + "observed failures per unit of exposure on this duty cycle, with the exposure " + "actually counted -- which requires the run and hour counting this module already " + "reports is missing" + ), + ), + Refusal( + quantity="remaining useful life", + why=( + "the honest version of this question is a distribution, not a number, and the " + "number is what gets used. An instrument reported as having 300 hours left will be " + "scheduled for 290 of them by somebody who never saw the interval around it" + ), + what_it_would_take=( + "a degradation signal measured on this instrument over time, plus a threshold " + "somebody validated against real end-of-life events on the same model" + ), + ), + Refusal( + quantity="probability a campaign completes without a stoppage", + why=( + "this is the number a planner most wants and the one least supportable here. It " + "multiplies per-instrument reliabilities nobody has measured and assumes they are " + "independent, which shared power, shared operators, and a shared environment make " + "false in a direction that flatters the answer" + ), + what_it_would_take=( + "measured per-instrument reliability on this duty cycle AND a characterized " + "correlation structure between instruments, which no published autonomous-lab work " + "reports" + ), + ), +) + + +def refusals() -> Tuple[Refusal, ...]: + """The predictions this layer declines to make, with what each would require. + + Worth returning rather than only documenting: a lab asking this module for a failure + rate gets an answer that names what it would have to go and measure, instead of silence + that invites somebody to write the estimate themselves. + """ + return NOT_COMPUTED + + +__all__ = [ + "Accrued", + "BoundaryReport", + "Campaign", + "Charge", + "Entitlement", + "InstrumentHealth", + "Interval", + "IntervalKind", + "NOT_COMPUTED", + "OBLIGATIONS", + "Refusal", + "Remaining", + "SERVICE_HISTORY", + "ServiceRecord", + "Standing", + "Untrusted", + "crosses_boundary", + "entitlement_summary", + "health_for", + "refusals", + "standing", + "untrusted_instruments", +] diff --git a/autonomous_lab/feedback.py b/autonomous_lab/feedback.py new file mode 100644 index 0000000..696bdd5 --- /dev/null +++ b/autonomous_lab/feedback.py @@ -0,0 +1,887 @@ +"""Whether a loop that claims to correct itself can actually close, and where it opens. + +Every other layer in this package plans a run or costs one. None of them models the run +ADJUSTING ITSELF, which is the thing the phrase "self-driving lab" is actually about. A +closed loop is three operations -- measure, compare, act -- and in a laboratory each one has +its own specific way of being fake: + + MEASURE the number arrives after the material it describes has already been consumed. + COMPARE there is no envelope to compare against, so nothing defines what good is. + ACT the correction is one the thing holding the loop may not make. + +A loop that fails any of the three is an open loop with a control diagram drawn around it. +This module makes a loop declare all three and computes whether it closes. + +The load-bearing idea is LATENCY versus ACTIONABILITY, and it is the one that does the +damage. A measurement taken after the correction point cannot steer anything -- not with a +better instrument, not with a faster instrument, not with a better model reading it. It is a +post-mortem wearing the costume of a control loop. So every loop declares where its +measurement is taken and where its correction would be applied, and any loop whose sensor +sits downstream of its actuator is refused outright. + +That refusal is pure graph reasoning over the protocol this repo already models. It needs no +new evidence, no instrument, and no decode, and it kills the two loops this lab would most +like to have: + + The genomics protocol quantifies its library on the plate reader AFTER the pool. The one + number that could size a pool arrives one step too late to size it. Repair the reader + tomorrow and the loop is still open, because what is wrong is the ORDER, and the fix is a + different assay at a different point in the protocol rather than a working instrument. + + Reading yield off the sequencing run to steer the sequencing run is the same shape with a + flow cell attached. It is a real and useful thing to do, and it is a NEXT-RUN loop rather + than a control loop. Calling it closed-loop control hides the part that matters, which is + that the current run is already spent. + +The other two refusals are separate on purpose and are never collapsed into the first, +because each one is bought with different work. NO_ENVELOPE means nothing in this repo says +what good is for that quantity, so there is no target to steer toward and the loop is +underspecified rather than late; the fix is a threshold with a basis, which costs an +experiment. UNAVAILABLE_MEASUREMENT is resolved from the ledger -- the step the number comes +from is blocked, manual, or broken -- and the fix is a decode, a debug, or a person. And +UNPERMITTED_CORRECTION means the loop declares a controller weaker than the correction +requires, which is settled by a decision rather than by money. Reporting one number over +four different budgets is the failure this whole package exists to refuse. + +The refusals are ordered so the headline is the one a reader must not be sent away from. A +lab told "the measurement is unavailable" will go buy an instrument, and if the loop is also +structurally open that instrument buys nothing at all. So the structural refusal is reported +first and the fixable ones after it, and every refusal a loop earns is carried, not just the +first, so nobody discovers the next one only after paying for the last. + +WHAT THIS MODULE WILL NOT SAY. It does not model gain, overshoot, damping, stability, or a +settling time. Those are properties of a plant, and computing any of them needs a plant model +plus measured response data: how far the output moves for a given change in the input, and +how long it takes to get there. This package has neither. `throughput` already refuses to +return plates per day because 18 of 18 steps have never been timed, and the same emptiness +sits underneath every dynamic quantity -- a settling time returned over unmeasured response +data would be a number that gets quoted, and then designed against. + +What protocol structure alone DOES support is a count. Between the sensor and the actuator +there are steps, and on a line running full there is material standing at each of them. Every +one of those plates reaches the correction point before the newest measurement can change it, +so the correction cannot help them however good it is. `in_flight_exposure` returns that +count. It is the honest form of "how fast can this loop react": in plates, which the protocol +knows, rather than in seconds, which nobody here has measured. + +Two scoping notes, because a verdict read wider than it was computed is worse than no +verdict. CLOSES is a claim about the LOOP -- its order, its target, its measurement, and its +authority. It is not a claim that the step being corrected runs at all; the ledger owns that +question, and `corrections_nobody_can_apply` reports it beside the verdict rather than inside +it, the same way qc keeps a wrong-assay gate separate from an unsatisfiable one. And the +envelope a loop steers toward is resolved from the qc criterion that already owns the number, +never restated here: two copies of one cutoff eventually disagree, and a loop steering toward +the retired copy would look exactly like a loop steering toward the live one. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Dict, List, Optional, Tuple + +from .model import Verdict +from .qc import GATES, MEASUREMENTS, PROTOCOL_GATES, Basis, Comparison, Criterion, Gate + +# The dynamic quantities this module refuses to emit, named so the refusal is explicit +# rather than merely absent. Each one needs a plant model and measured response data -- +# how much the output moves per unit of input, and over what horizon -- and this package +# has never measured a single step duration, let alone a response curve. Naming them here +# is what makes the omission checkable instead of a thing somebody adds later by accident. +NOT_MODELLED: Tuple[str, ...] = ( + "gain", + "overshoot", + "damping", + "stability margin", + "time constant", + "settling time", +) + + +class Controller(str, Enum): + """Who computes the correction and applies it. Ordered weakest to strongest. + + The ordering is by how little a generated token can move, not by how much intelligence is + applied, and that is what makes it computable. A drafting model is still in the path: the + ratifier reads a proposal that has already framed the problem, and a plausible frame is + what a model produces most reliably when it is wrong. + + This names the same distinction the authority layer draws over decision classes, and it is + restated rather than imported on purpose -- a loop's controller is a property of the loop, + the ordering here is short enough to read in one screen, and this module has to be + checkable on its own. If the two are ever wired together, the authority layer owns the + requirement and this resolves from it rather than keeping a second copy. + """ + + MODEL = "model" # a model emits the correction and the correction is applied + MODEL_PROPOSES = "model_proposes" # a model drafts; a coded check or a person ratifies + CODED = "coded" # a coded controller computes and applies it; no model in the path + HUMAN = "human" # a named person decides and performs it, and is accountable for it + + @property + def model_in_path(self) -> bool: + """True where model output reaches the correction at all, deciding or drafting. + + This is the line the enum exists to draw. A correction that consumes material does not + become safe by adding a reviewer to a model's account of why it should be consumed. + """ + return self in (Controller.MODEL, Controller.MODEL_PROPOSES) + + @property + def rank(self) -> int: + return _CONTROLLER_ORDER.index(self) + + def may_apply(self, correction: "Correction") -> bool: + """Is this controller strong enough for what the correction actually does? + + Strengthening always satisfies. A lab may put a person on a correction a coded + controller could make; the reverse is the defect this check exists to catch. + """ + return self.rank >= correction.minimum_controller.rank + + +# Weakest first. Used by rank, may_apply, and every report that orders by strength. +_CONTROLLER_ORDER: Tuple[Controller, ...] = ( + Controller.MODEL, + Controller.MODEL_PROPOSES, + Controller.CODED, + Controller.HUMAN, +) + + +class Correction(str, Enum): + """What the correction does to the process, which is what decides who may apply it. + + Deliberately not qc.Decision, which is the neighbouring vocabulary. A gate decides what to + do with a plate that already failed; a control loop changes a setting so the next one does + not. RETUNE has no Decision equivalent at all, and that absence is the tell that these are + two vocabularies rather than one restated -- reusing Decision here would have forced every + parameter change to be reported as a RETRY, which is a different physical act. + """ + + HOLD = "hold" # stop the line where it is and wait for someone + RETUNE = "retune" # change a parameter the next execution of the step will run under + REPEAT = "repeat" # run the step again on material that is still viable + DIVERT = "divert" # route the material down a different path + DISCARD = "discard" # destroy material, or spend a scarce input to salvage it + + @property + def reversible(self) -> bool: + """Can the material be put back the way it was after a wrong correction? + + True only for HOLD, and the asymmetry is the point rather than an artifact of a short + list. Stopping commits nothing: the plate is where it was and a wrong stop costs time. + Everything else spends something before anybody reads the decision that spent it, which + is why the strength required climbs with the spend rather than with the difficulty. + """ + return self is Correction.HOLD + + @property + def minimum_controller(self) -> Controller: + return _MINIMUM_CONTROLLER[self] + + +# Why each floor sits where it does. These are arguments, not preferences, and each one +# points at something already computed elsewhere in this package. +_MINIMUM_CONTROLLER: Dict[Correction, Controller] = { + # Stopping is the one correction a model may make alone, because the failure of a wrong + # stop is a delay somebody notices immediately. Setting this any higher would mean a lab + # whose model saw a problem had to wait for a ratifier before the line stopped. + Correction.HOLD: Controller.MODEL, + # A parameter change is quiet and cumulative. This repo's own volume_short_within_tolerance + # failure is exactly a parameter error that nothing flags, biases the dataset instead of + # breaking it, and stays inside the instrument's tolerance -- so a draft is useful and it + # is not the last word. + Correction.RETUNE: Controller.MODEL_PROPOSES, + # Repeating spends a reagent set, and the qc layer already separates RETRY from RECOVER + # because material that changed state is destroyed by a repeat rather than rescued by one. + # Nothing in this module can tell those apart, so a ratifier must. + Correction.REPEAT: Controller.MODEL_PROPOSES, + # Routing material is where attribution dies. The lineage layer's whole finding is that a + # wrong merge is permanent at the moment it happens, and an interlock reading a barcode + # cannot be argued into a different route by a well-formed account of the situation. + Correction.DIVERT: Controller.CODED, + # Destroying a sample cannot be undone by noticing afterward that the correction was + # wrong. A named person approves it and is accountable for having approved it. + Correction.DISCARD: Controller.HUMAN, +} + + +class Closable(str, Enum): + """Whether a control loop can close, and if not, which of the three legs is fake. + + The four refusals are listed in the order they are reported, and that order is the line + this enum draws. OPEN_LOOP_SENSOR_TOO_LATE is first because it is the only one no + purchase, decode, or policy change repairs: the protocol has to be rewritten so the + measurement happens before the thing it steers. Reporting a cheaper reason ahead of it + sends a lab to buy an instrument for a loop that could never have closed. + """ + + OPEN_LOOP_SENSOR_TOO_LATE = "open_loop_sensor_too_late" # measured at or after the correction point + NO_ENVELOPE = "no_envelope" # nothing declares what good is; there is nothing to steer toward + UNAVAILABLE_MEASUREMENT = "unavailable_measurement" # the measuring step does not produce a number here + UNPERMITTED_CORRECTION = "unpermitted_correction" # the controller is weaker than the correction needs + CLOSES = "closes" + + @property + def closed(self) -> bool: + return self is Closable.CLOSES + + @property + def structural(self) -> bool: + """True only for the refusal that money cannot move. + + The other three name work a lab can buy, schedule, or decide. This one names an ordering + in the protocol, and the only thing that changes it is measuring somewhere else. + """ + return self is Closable.OPEN_LOOP_SENSOR_TOO_LATE + + +# Reported in this order. CLOSES is not in it: it is what is left when nothing else applies. +_REFUSAL_ORDER: Tuple[Closable, ...] = ( + Closable.OPEN_LOOP_SENSOR_TOO_LATE, + Closable.NO_ENVELOPE, + Closable.UNAVAILABLE_MEASUREMENT, + Closable.UNPERMITTED_CORRECTION, +) + + +# Which verdicts mean a number actually arrives from a step. Identical to the partition +# qc.Readiness draws over the same verdicts, and taken here off the ledger row the ledger +# already computed rather than by costing the step a second time. A step that is blocked, +# manual, written, or broken produces nothing to compare against, and a loop reading one is +# steering on a value nobody will ever hand it. +_MEASUREMENT_ARRIVES: Tuple[Verdict, ...] = (Verdict.AUTOMATED, Verdict.SUPERVISED) + + +@dataclass(frozen=True) +class Envelope: + """The band a correction steers the measured quantity back into. + + An envelope with no bound on either side is refused at construction. That is the vacuous + case and it is the dangerous one: an unbounded band contains every value, so a loop built + on it reports itself in specification forever, corrects nothing, and looks exactly like a + loop that is working. Absence of a bound is not a wide bound. + + `basis` comes from the qc criterion this was resolved from and is carried rather than + checked. An envelope resting on intuition is still an envelope and the loop can still + close around it -- what it cannot do is pass unreported, so `unvalidated_envelopes()` + lists it. + """ + + quantity: str # the qc.Measurement key this bounds + units: str + lower: Optional[float] = None + upper: Optional[float] = None + basis: Basis = Basis.INTUITION + note: str = "" + + def __post_init__(self) -> None: + if self.lower is None and self.upper is None: + raise ValueError( + f"envelope for '{self.quantity}' bounds nothing on either side; an unbounded band " + "accepts every value, so a loop steering toward it never corrects anything" + ) + if self.lower is not None and self.upper is not None and self.upper < self.lower: + raise ValueError( + f"envelope for '{self.quantity}' has upper {self.upper} below lower {self.lower}" + ) + + @property + def validated(self) -> bool: + return self.basis.validated + + def contains(self, value: float) -> bool: + if self.lower is not None and value < self.lower: + return False + if self.upper is not None and value > self.upper: + return False + return True + + def describe(self) -> str: + if self.lower is not None and self.upper is not None: + return f"{self.lower} <= {self.quantity} <= {self.upper} {self.units}" + if self.lower is not None: + return f"{self.quantity} >= {self.lower} {self.units}" + return f"{self.quantity} <= {self.upper} {self.units}" + + +def envelope_from_criterion(gate: Gate, criterion_name: str) -> Envelope: + """Build an envelope out of a threshold the qc layer already owns. + + The number, the direction, the quantity it bounds, and the basis it rests on are all qc's + facts. Restating any of them here would give this lab two copies of one cutoff, and the + day somebody revises the gate the loop would go on steering toward the retired band with + nothing anywhere to reveal which of the two was live. Units are resolved the same way, off + the declared measurement, so an envelope cannot be built for a quantity qc does not know. + """ + match: Optional[Criterion] = next((c for c in gate.criteria if c.name == criterion_name), None) + if match is None: + raise KeyError( + f"gate '{gate.name}' declares no criterion '{criterion_name}'; " + f"it has: {[c.name for c in gate.criteria]}" + ) + measurement = MEASUREMENTS.get(match.measurement) + if measurement is None: + raise KeyError( + f"criterion '{criterion_name}' reads '{match.measurement}', which qc does not declare " + "as a measurement; an envelope over an undeclared quantity bounds nothing" + ) + lower: Optional[float] = None + upper: Optional[float] = None + if match.comparison is Comparison.AT_LEAST: + lower = match.bound + elif match.comparison is Comparison.AT_MOST: + upper = match.bound + else: + lower, upper = match.bound, match.upper + return Envelope( + quantity=match.measurement, + units=measurement.units, + lower=lower, + upper=upper, + basis=match.basis, + note=f"resolved from qc gate '{gate.name}', criterion '{match.name}'", + ) + + +@dataclass(frozen=True) +class Loop: + """One control loop: what it watches, where it watches it, and what it changes. + + There is deliberately no field here that says whether the loop closes. Closability is + computed against a real protocol and a real ledger, the same way a step's verdict is, and + a loop that could declare itself closed would be a loop whose author could silence the + only report in this module worth reading. + + `measured_at` and `corrects` are Step.op values resolved against the protocol, so a loop + cannot name a step that does not exist, and a loop naming an op that appears twice is + refused rather than resolved to the first one: an op used at two points in a flow is not a + position, and guessing which was meant is how a loop reads as closed because the wrong + occurrence happened to sit upstream. + """ + + name: str + quantity: str # the measured quantity; a qc.Measurement key wherever one exists + measured_at: str # Step.op where the number is taken + corrects: str # Step.op the correction is applied to + correction: Correction + applied_by: Controller + envelope: Optional[Envelope] = None + note: str = "" + + def permitted(self) -> bool: + """Is the declared controller strong enough for the declared correction? + + A method rather than a field. A stored answer sitting beside the controller and the + correction it was derived from eventually disagrees with them, and nothing on the object + reveals which of the three is wrong. + """ + return self.applied_by.may_apply(self.correction) + + def steers_toward(self) -> str: + if self.envelope is None: + return "nothing; no envelope is declared for this quantity" + return self.envelope.describe() + + +@dataclass(frozen=True) +class Blocker: + """One reason a loop does not close, with the specific thing that is wrong.""" + + reason: Closable + detail: str + + +@dataclass(frozen=True) +class Closure: + """A loop costed against a real protocol and a real ledger. + + `blockers` is every refusal the loop earned, in reporting order, and the verdict is the + first of them. Carrying all of them matters more than it looks: a loop is usually broken + in more than one way, and a report that showed only the first would send a lab to fix that + one and rediscover the next after paying for it. + + `corrects_verdict` is the ledger's verdict for the step being corrected, carried here so + no reader has to look it up separately. It is NOT part of the verdict, and that scoping is + deliberate -- see `FeedbackReport.corrections_nobody_can_apply`. + """ + + loop: Loop + blockers: Tuple[Blocker, ...] + measured_index: int + corrects_index: int + corrects_verdict: Verdict + + @property + def verdict(self) -> Closable: + return self.blockers[0].reason if self.blockers else Closable.CLOSES + + @property + def closes(self) -> bool: + return not self.blockers + + @property + def reason(self) -> str: + if self.blockers: + return self.blockers[0].detail + return ( + f"'{self.loop.quantity}' is measured at '{self.loop.measured_at}', which is upstream of " + f"'{self.loop.corrects}'; it steers toward {self.loop.steers_toward()}, the number " + f"arrives, and {self.loop.applied_by.value} may make a {self.loop.correction.value}" + ) + + def all_reasons(self) -> Tuple[str, ...]: + return tuple(b.detail for b in self.blockers) + + def refusals(self) -> Tuple[Closable, ...]: + return tuple(b.reason for b in self.blockers) + + +@dataclass(frozen=True) +class InFlight: + """How much material is committed between a loop's sensor and its actuator. + + Counted in steps and reported in plates, never in seconds. Seconds would need step + durations, and `throughput` already refuses to return a plates-per-day figure because no + step in this repo has ever been timed. The same emptiness would sit under any reaction + time reported here, so this reports the thing the protocol actually knows. + """ + + loop: Loop + steps_between: int + intervening: Tuple[str, ...] + meaning: str + + @property + def immediate(self) -> bool: + """True only when the correction lands at the very next step. + + The single case where a correction reaches every plate it was computed for. Anything + else means material passes the correction point under the setting the correction is + about to change, and no amount of controller quality reaches it. + """ + return self.steps_between == 0 + + +def _index_of(protocol, op: str, role: str) -> int: + """Where a step op sits in the protocol's order, refusing anything ambiguous.""" + hits = [i for i, step in enumerate(protocol.steps) if step.op == op] + if not hits: + raise KeyError( + f"loop names '{op}' as its {role}, which is not a step in protocol '{protocol.name}'" + ) + if len(hits) > 1: + raise ValueError( + f"'{op}' appears {len(hits)} times in protocol '{protocol.name}' (positions " + f"{hits}), so it is not a position and cannot be a loop's {role}. Picking the first " + "occurrence would report a loop as closed whenever the wrong one happened to sit " + "upstream" + ) + return hits[0] + + +def can_close(loop: Loop, protocol, ledger) -> Closure: + """Can this loop actually close against this protocol and this workcell? + + Step ordering is resolved from the protocol and measurement availability from the ledger. + Neither is re-derived: the protocol owns the order the bench runs in, the ledger owns + whether a step produces anything, and a second computation of either would eventually + disagree with the first with nothing to say which was right. + + Every check runs. The function does not return early on the first refusal, because a loop + broken three ways that reported one way would be fixed one way and still be broken. + """ + if ledger.protocol.name != protocol.name: + raise ValueError( + f"ledger was built for protocol '{ledger.protocol.name}' but the loop is being costed " + f"against '{protocol.name}'; a verdict mixed from two protocols describes neither" + ) + + measured_index = _index_of(protocol, loop.measured_at, "measured_at") + corrects_index = _index_of(protocol, loop.corrects, "corrects") + blockers: List[Blocker] = [] + + # -- 1. can the measurement reach the material it is supposed to steer? ------ + # The one refusal no purchase moves, so it is checked and reported first. + if measured_index > corrects_index: + blockers.append( + Blocker( + Closable.OPEN_LOOP_SENSOR_TOO_LATE, + f"'{loop.quantity}' is measured at '{loop.measured_at}' (step {measured_index + 1}), " + f"which is downstream of '{loop.corrects}' (step {corrects_index + 1}). The material " + "has already passed the correction point when the number arrives, so this steers a " + "future run and not this one. That is a next-run adjustment, not a control loop, and " + "no instrument, decode, or model changes it -- only measuring earlier does", + ) + ) + elif measured_index == corrects_index: + blockers.append( + Blocker( + Closable.OPEN_LOOP_SENSOR_TOO_LATE, + f"'{loop.measured_at}' is both the sensor and the actuator. This package models a " + "step as atomic, so it cannot see a correction applied inside one and will not claim " + "a loop closes on evidence it does not have. Declare the measurement as its own step " + "if the instrument really reports mid-operation", + ) + ) + + # -- 2. is there anything to compare against? -------------------------------- + # An envelope bounding some other quantity is not an envelope for this one. That case + # looks like a fully specified loop in every summary that counts declarations. + envelope = loop.envelope + if envelope is None: + blockers.append( + Blocker( + Closable.NO_ENVELOPE, + f"nothing declares what a good '{loop.quantity}' is, so there is no band to steer " + "toward and no way to tell a correction that helped from one that did not. This is " + "not a missing instrument; it is a missing threshold, and it costs an experiment and " + "a stated basis", + ) + ) + elif envelope.quantity != loop.quantity: + blockers.append( + Blocker( + Closable.NO_ENVELOPE, + f"the declared envelope bounds '{envelope.quantity}', not '{loop.quantity}'. A band " + "on a neighbouring quantity is not a target for this one, and it counts as a " + "declaration in every summary that only counts whether an envelope is present", + ) + ) + + # -- 3. does the number actually arrive? ------------------------------------- + measuring_row = ledger.rows[measured_index] + if measuring_row.verdict not in _MEASUREMENT_ARRIVES: + blockers.append( + Blocker( + Closable.UNAVAILABLE_MEASUREMENT, + f"'{loop.quantity}' comes from '{loop.measured_at}', which this workcell costs as " + f"{measuring_row.verdict.value}: {measuring_row.reason}. A loop over a number nobody " + "produces is an open loop that reports no error, because the comparison never runs", + ) + ) + + # -- 4. may the declared controller make this correction? -------------------- + if not loop.permitted(): + blockers.append( + Blocker( + Closable.UNPERMITTED_CORRECTION, + f"a {loop.correction.value} correction needs at least " + f"{loop.correction.minimum_controller.value} and this loop declares " + f"{loop.applied_by.value}" + + ( + ". The correction is reversible, so the floor is low and this is still below it" + if loop.correction.reversible + else ". The correction cannot be undone by noticing afterward that it was wrong" + ), + ) + ) + + blockers.sort(key=lambda b: _REFUSAL_ORDER.index(b.reason)) + return Closure( + loop=loop, + blockers=tuple(blockers), + measured_index=measured_index, + corrects_index=corrects_index, + corrects_verdict=ledger.rows[corrects_index].verdict, + ) + + +def in_flight_exposure(loop: Loop, protocol) -> InFlight: + """How many plates are already committed between the sensor and the correction point. + + This is the only thing this module will say about how fast a loop reacts, and it is said + in plates because plates are what the protocol knows. Every step between the two holds + material on a line running full, and all of it reaches the correction point before the + newest measurement can change anything there. A perfect controller does not help those + plates, which makes this a property of the protocol rather than of the controller. + + Refused for a loop whose sensor is not upstream of its actuator. There is no in-flight + count for a correction that never reaches its own material, and returning zero would read + as the best possible case rather than as the absence of a loop. + """ + measured_index = _index_of(protocol, loop.measured_at, "measured_at") + corrects_index = _index_of(protocol, loop.corrects, "corrects") + if measured_index >= corrects_index: + raise ValueError( + f"loop '{loop.name}' measures at or after the step it corrects, so no material is in " + f"flight between them -- the correction never reaches this run's material at all. " + f"can_close refuses it as {Closable.OPEN_LOOP_SENSOR_TOO_LATE.value}" + ) + intervening = tuple(step.op for step in protocol.steps[measured_index + 1 : corrects_index]) + count = len(intervening) + if count == 0: + meaning = ( + f"the correction lands at '{loop.corrects}', the step immediately after the " + f"measurement. Nothing is committed in between, so every plate this loop measures is a " + "plate it can still steer. This is the tightest loop the protocol admits" + ) + else: + meaning = ( + f"{count} step position(s) sit between '{loop.measured_at}' and '{loop.corrects}': " + f"{', '.join(intervening)}. On a line running full that is {count} plate(s) already " + f"past the sensor and not yet at the correction point; each was admitted before this " + f"measurement existed and reaches '{loop.corrects}' under the setting the correction " + "is about to change. No controller helps them. Nothing here converts that into " + "seconds: no step in this package has ever been timed, and throughput refuses the same " + "conversion for the same reason" + ) + return InFlight(loop=loop, steps_between=count, intervening=intervening, meaning=meaning) + + +# -- the loops this lab would want, costed -------------------------------------- +# Written as a lab would actually propose them, which is why most of them fail. Each names a +# real quantity in the reference protocols and a real step it would change. + +LIBRARY_CONCENTRATION = envelope_from_criterion( + GATES["library_quant_before_flow_cell"], "library_present" +) +LIBRARY_PURITY = envelope_from_criterion( + GATES["library_quant_before_flow_cell"], "not_protein_contaminated" +) +SORT_OCCUPANCY = envelope_from_criterion( + GATES["sort_occupancy_before_amplification"], "enough_wells_occupied" +) + + +LOOPS: Dict[str, Loop] = {} + +for _loop in ( + Loop( + name="pooled_quant_steers_pooling", + quantity="library_conc_od", + measured_at="read_absorbance", + corrects="library_pool", + correction=Correction.RETUNE, + applied_by=Controller.CODED, + envelope=LIBRARY_CONCENTRATION, + note=( + "the loop everybody draws first and the clearest open loop in this repo. Normalizing a " + "pool is exactly what a concentration measurement is for, and this protocol measures " + "after the pool, so the number describes material whose composition is already fixed. " + "The order is what is wrong, not the reader: repairing the Tecan leaves this open, and " + "quantifying per well before pooling closes it with no decode at all" + ), + ), + Loop( + name="quant_steers_flow_cell_loading", + quantity="library_conc_od", + measured_at="read_absorbance", + corrects="upload_manifest", + correction=Correction.RETUNE, + applied_by=Controller.CODED, + envelope=LIBRARY_CONCENTRATION, + note=( + "structurally sound and the one worth fixing: the quant genuinely does precede the " + "manifest, so a working reader would close it. What it steers toward is another " + "matter, and qc already reports it -- A260 at picogram input is precise and about the " + "wrong thing, so a closed loop here would confidently load an empty library" + ), + ), + Loop( + name="purity_steers_flow_cell_loading", + quantity="library_purity_260_280", + measured_at="read_absorbance", + corrects="set_run_parameters", + correction=Correction.HOLD, + applied_by=Controller.MODEL, + envelope=LIBRARY_PURITY, + note=( + "the weakest correction in the set and the only one a model may make alone, because " + "stopping before a flow cell is committed spends nothing but time. It still does not " + "close, and for a reason that has nothing to do with the model" + ), + ), + Loop( + name="occupancy_steers_amplification", + quantity="well_occupancy", + measured_at="start_sort", + corrects="wgs_prep_lysis", + correction=Correction.DIVERT, + applied_by=Controller.MODEL_PROPOSES, + envelope=SORT_OCCUPANCY, + note=( + "the loop with the best shape in this repo -- the sort is genuinely upstream of the " + "amplification, and skipping empty wells is worth a reagent set. It fails twice, and " + "the two failures are bought separately: the dispenser's command set is undecoded, and " + "routing material past a step is not something a drafting model should do unratified" + ), + ), + Loop( + name="yield_steers_resequencing", + quantity="run_yield_pf", + measured_at="watch_run_folder", + corrects="start_run", + correction=Correction.REPEAT, + applied_by=Controller.MODEL_PROPOSES, + envelope=Envelope( + quantity="run_yield_pf", + units="Gbp", + lower=1.0, + basis=Basis.INTUITION, + note=( + "a floor on usefulness rather than a specification. Deliberately not resolved from a " + "qc criterion, because no gate in this repo reads run yield -- and an envelope with " + "no gate behind it is exactly the kind that never gets revised" + ), + ), + note=( + "what a lab means when it says the system learns from its runs. It is real and it is " + "useful and it is not control: the yield is read off a run that is already finished, " + "so the flow cell it would have saved is spent. Reporting it as a closed loop is how a " + "next-run adjustment gets budgeted as a safeguard" + ), + ), + Loop( + name="evaporation_steers_injection", + quantity="residual_solvent_mass", + measured_at="get_status", + corrects="set_injection", + correction=Correction.RETUNE, + applied_by=Controller.CODED, + envelope=None, + note=( + "the chemistry loop, and the one that shows what a missing envelope actually looks " + "like. Its order is fine. What is absent is a target: qc declares no gate at all for " + "this protocol, because nothing reads the chromatogram back, so there is no number in " + "this repo that says what a good dry-down is. A loop here would run, act, and have no " + "way to be wrong" + ), + ), +): + LOOPS[_loop.name] = _loop + + +# Keyed by protocol so a protocol with no declared loops is visibly unlooped rather than +# silently so, matching how qc keys its gates. +PROTOCOL_LOOPS: Dict[str, Tuple[str, ...]] = { + "single_cell_genomics": ( + "occupancy_steers_amplification", + "pooled_quant_steers_pooling", + "quant_steers_flow_cell_loading", + "purity_steers_flow_cell_loading", + "yield_steers_resequencing", + ), + "small_molecule_qc": ("evaporation_steers_injection",), +} + + +@dataclass +class FeedbackReport: + """Every declared loop on a protocol, costed against a real workcell.""" + + protocol: str + rows: List[Closure] + + def closable(self) -> List[Closure]: + return [r for r in self.rows if r.closes] + + def refused(self) -> List[Closure]: + return [r for r in self.rows if not r.closes] + + def structurally_open(self) -> List[Closure]: + """Loops whose sensor is not upstream of their actuator. + + Reported on its own because it is the group whose fix is a protocol change. Every other + refusal in this module is bought with work; this one is bought with a redesign, and + folding them together would put both budgets behind one number. + """ + return [r for r in self.rows if r.verdict.structural] + + def by_reason(self) -> Dict[str, List[Closure]]: + out: Dict[str, List[Closure]] = {} + for row in self.rows: + out.setdefault(row.verdict.value, []).append(row) + return out + + def unvalidated_envelopes(self) -> List[Closure]: + """Loops steering toward a band this lab has not earned with its own data. + + A caveat and not a refusal, deliberately. A loop around a vendor or intuition threshold + genuinely closes -- it drives the quantity into that band, reliably, whether or not the + band is the right one. Making it a fifth refusal would say the fix is the same as for a + loop with no target at all, and it is not: this one needs a titration, not a threshold. + """ + return [ + r for r in self.rows if r.loop.envelope is not None and not r.loop.envelope.validated + ] + + def corrections_nobody_can_apply(self) -> List[Closure]: + """Loops whose correction lands on a step no code path reaches, with no human declared. + + Kept out of the verdict on purpose, the same way qc keeps a wrong-assay gate out of + readiness. Whether a step can be performed is the ledger's question and it is already + answered there; re-deciding it inside a closure verdict would put an execution problem + and a control problem behind one word. A human-applied correction at a manual step is a + perfectly good closed loop, just a slow one, so only a declared machine controller + counts here. + """ + return [ + r + for r in self.rows + if r.loop.applied_by is not Controller.HUMAN + and r.corrects_verdict not in _MEASUREMENT_ARRIVES + ] + + def counts(self) -> Dict[str, int]: + out = {reason.value: 0 for reason in Closable} + for row in self.rows: + out[row.verdict.value] += 1 + out["total"] = len(self.rows) + return out + + def closes_any(self) -> bool: + """True only if at least one declared loop actually closes. + + Not `all`. A lab does not need every proposed loop to close, and a report that demanded + it would flatten the interesting result -- which is which loops close and which do not. + """ + return bool(self.closable()) + + +def feedback_report(protocol, ledger) -> FeedbackReport: + """Cost every loop declared for a protocol against the ledger built for it.""" + names = PROTOCOL_LOOPS.get(protocol.name, ()) + return FeedbackReport( + protocol=protocol.name, + rows=[can_close(LOOPS[name], protocol, ledger) for name in names], + ) + + +def loops_for(protocol_name: str) -> Tuple[Loop, ...]: + return tuple(LOOPS[name] for name in PROTOCOL_LOOPS.get(protocol_name, ())) + + +def ungated_protocols() -> Tuple[str, ...]: + """Protocols with declared loops and no qc gates behind them. + + Resolved from qc rather than restated, so it empties itself the day somebody writes a gate + for the chemistry flow. A protocol in this list cannot supply an envelope to any loop over + it, which is why its loops refuse with NO_ENVELOPE rather than with anything instrumental. + """ + return tuple( + name for name in PROTOCOL_LOOPS if not PROTOCOL_GATES.get(name, ()) + ) + + +__all__ = [ + "Blocker", + "Closable", + "Closure", + "Controller", + "Correction", + "Envelope", + "FeedbackReport", + "InFlight", + "LOOPS", + "Loop", + "NOT_MODELLED", + "PROTOCOL_LOOPS", + "can_close", + "envelope_from_criterion", + "feedback_report", + "in_flight_exposure", + "loops_for", + "ungated_protocols", +] diff --git a/autonomous_lab/teaching.py b/autonomous_lab/teaching.py new file mode 100644 index 0000000..5e8392c --- /dev/null +++ b/autonomous_lab/teaching.py @@ -0,0 +1,914 @@ +"""What an expert demonstrated, what the machine did, and whether those are the same thing. + +`intelligence` records what a senior scientist KNOWS: judgments as prose, benchmarks as +targets. Neither of those is the transfer. A judgment is knowledge written down and a +benchmark is a number somebody asserted; the step between "the robot executes the script" +and "the robot performs the task as well as the person who taught it" is a DEMONSTRATION -- +an expert performing the operation, the outcome measured, and the machine's performance +placed against it. That comparison is the actual content of automating expert work, and it +is the one thing nobody records. + +A DEMONSTRATION IS DATA, NOT AUTHORITY. This is the discipline of the whole module and it +cuts against how demonstrations are normally used, which is as a specification: the expert +did it this way, so this way is correct. A scientist doing something twice is not a +specification. It is two observations with a spread, and the SPREAD is the information -- +it is the only thing that separates how much of the difference between a good plate and a +bad one is the operation and how much is the day. A single demonstration establishes a +value and no tolerance at all, and a tolerance invented around it -- plus or minus ten +percent, plus or minus two sigma of a distribution nobody sampled -- would be the fabricated +number this repo refuses everywhere else. So `Envelope` refuses to state a tolerance below +MIN_DEMONSTRATIONS, and reports the refusal rather than widening quietly. + +The tolerance, where there is one, is the observed range and nothing else. No multiplier, no +assumed distribution, no smoothing. The min and max of what actually happened is a fact; +mean plus k standard deviations is a fact plus a k somebody chose, and k is exactly the kind +of number that reaches a report with no author attached to it. + +ONE GOOD MACHINE RUN IS NOT PARITY, which is why `Attainment` has four values rather than +three. A machine that ran the operation once, well, has produced a number that sits inside +the expert's range; reporting that as MEETS claims the machine performs like the person, on +evidence that cannot distinguish it from luck. The result carries no more information than +not having measured, and INDISTINGUISHABLE_FROM_UNMEASURED says so in its name rather than +in a footnote nobody reads. The same value covers the mirror case, where the machine has run +plenty and the expert side is too thin to place it against: either way there are numbers on +the page and they mean nothing yet. Parity is also judged on the machine's WORST observation +rather than its mean, because a mean lets one excellent run pay for a bad one, and at the +bench the bad plate is the one that costs the sample. + +BENCHMARK OR ENVELOPE. `intelligence.Benchmark` and `Envelope` look like one object and are +two kinds of evidence. A benchmark is a target somebody ASSERTED -- a CV below 5 percent, a +recovery inside specification. An envelope is a target somebody DEMONSTRATED -- the range a +competent person actually produced, on this material, on this bench. As evidence that a +number is ACHIEVABLE the envelope is strictly stronger, and the reason is arithmetic rather +than rhetoric: an assertion has no n. It cannot be wrong about the spread because it never +states one, and a claim that cannot be wrong is not evidence. As a statement of what the +science NEEDS the benchmark is the stronger one, and no quantity of demonstrations replaces +it -- an expert can demonstrate 40 percent recovery repeatably and 40 percent can still be +too little for the assay. They answer different questions and a lab needs both. What it must +not do is let one stand in for the other, so `transfer_report` carries them side by side and +resolves the benchmark from `intelligence` rather than restating its target here. + +THE BACKLOG NOBODY WRITES DOWN. `taught()` answers whether an operation has an expert +envelope at all, and in this repo the answer is no for every operation, because there is not +one recorded demonstration anywhere in it. That is not an omission in this module. It is the +state of the evidence, and writing it down converts it into the queue of what an SME still +has to demonstrate -- the real backlog of an automation programme, and the one nobody keeps, +because it is the only part of the plan that cannot be worked on by the people building the +automation. `demonstration_queue` ranks it the way `ledger.unlocks` ranks decoding work: by +what one sitting would free, rather than by what is convenient to schedule. + +The machine side is empty here for a reason worth stating. This lab has timed exactly one +mechanism, and that measurement lives in `throughput.DURATIONS`, which records a value and a +worst case rather than the series behind them. A summary cannot supply an n, so it cannot be +placed against an envelope, and copying its numbers into this module would be a second copy +of a fact a sibling already owns. How a lab records its measurements decides which questions +it can answer later, and that is a finding rather than a formatting complaint. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Dict, List, Optional, Sequence, Tuple + +from .intelligence import BENCHMARKS_BY_OP, Benchmark + +# How many demonstrations of one operation before a tolerance may be stated at all. Three is +# a floor set by convention rather than by a power calculation, and the module reports the +# number it used instead of burying it: two points have exactly one degree of freedom, so +# the interval they imply IS the two points, and nothing in them can say whether either is +# an outlier. The same minimum applies to the machine side, because the question is whether +# two spreads overlap and a single number has no spread to compare. Raising this would be +# defensible. Having none is not. +MIN_DEMONSTRATIONS = 3 + + +class Goal(str, Enum): + """Which way is better for a metric, which decides what leaving the range means. + + Without this the comparison has to pick a direction, and the natural pick -- inside the + range is good, outside is bad -- silently fails a machine that is more accurate than the + person who taught it. The reverse error is worse: on a delivered volume, overshooting the + expert by the same margin that would fail on the low side is not better performance, it is + a different failure, and a one-sided comparison passes it. + """ + + HIGHER = "higher" # more is better: recovery, occupancy, yield + LOWER = "lower" # less is better: CV, carryover, residual volume + WINDOW = "window" # a target with two sides: delivered volume against nominal + + @property + def outside_is_always_worse(self) -> bool: + """True when leaving the demonstrated range in either direction is a failure.""" + return self is Goal.WINDOW + + +class Attainment(str, Enum): + """How a machine's performance places against what an expert demonstrated. + + Four values because three would force the interesting case into the wrong bucket. MEETS + and BELOW are claims about performance and both need enough observations to be made at + all. UNMEASURED means one side of the comparison is empty. The fourth means the numbers + exist and cannot yet distinguish the machine from a machine nobody measured, which is the + state most demos are in and the state no demo reports itself as being in. + """ + + MEETS = "meets" # enough machine observations, and the worst of them attains the envelope + BELOW = "below" # enough machine observations, and they do not attain it + UNMEASURED = "unmeasured" # one side has no observations at all; nothing to compare + INDISTINGUISHABLE_FROM_UNMEASURED = "indistinguishable_from_unmeasured" # too few to place + + @property + def demonstrated_parity(self) -> bool: + """True only when a machine has been SHOWN to perform as well as the person. + + The other three are not degrees of parity, they are three ways of not knowing, and a + report that treated them as a scale would rank ignorance against evidence. + """ + return self is Attainment.MEETS + + +@dataclass(frozen=True) +class Demonstration: + """One expert, performing one operation once, with the outcome measured. + + `by` is a stable handle for the demonstrator rather than a name, and it is here because + three demonstrations from one person and three from three people are not the same + evidence. The first describes that person; only the second begins to describe the + operation. Pooling them without recording which is which is how an envelope ends up + encoding one individual's habits and calling it a standard. + + `conditions` is what would have to match for this demonstration to say anything about + another run -- the material, the input amount, the labware. A demonstration at nanogram + input is not evidence about picogram input, and the whole difficulty of low-input work is + that the two look like the same operation. + """ + + operation: str + metric: str + units: str + value: float + by: str # demonstrator handle, not a name + conditions: str + evidence: str = "" + + +@dataclass(frozen=True) +class MachineObservation: + """One machine run of an operation, measured on the same metric as the expert. + + Deliberately a separate type from `Demonstration` despite carrying the same fields. They + are different evidence about different agents, and one container for both invites the + arithmetic that pools them -- which would compute a range over expert and machine runs + together and report the machine as being inside it. + """ + + operation: str + metric: str + units: str + value: float + by: str # the run card, script, or instrument that produced it + conditions: str + evidence: str = "" + + +@dataclass(frozen=True) +class Envelope: + """What "as good as the expert" means for one operation, derived from demonstrations. + + Nothing here is declared. There is no field an author can set to state a tolerance, a + center, or a spread, because a derived value kept beside the values it derives from + eventually disagrees with them and nothing reveals which one is wrong. Every number this + class reports is a method over `demonstrations`, and an envelope with too few of them + returns None and says why rather than returning a plausible interval. + + `goal` and the metric come from the operation's `Transferable` spec, not from the + demonstrations, so a demonstration cannot redefine what better means by being recorded. + """ + + operation: str + metric: str + units: str + goal: Goal + demonstrations: Tuple[Demonstration, ...] = () + + def __post_init__(self) -> None: + for d in self.demonstrations: + if d.operation != self.operation: + raise ValueError( + f"envelope for '{self.operation}' holds a demonstration of '{d.operation}'" + ) + if d.metric != self.metric or d.units != self.units: + raise ValueError( + f"envelope for '{self.operation}' measures {self.metric} in {self.units}, but a " + f"demonstration by {d.by} records {d.metric} in {d.units}; two metrics pooled into " + "one range is a number about nothing" + ) + + # -- derived, and therefore methods ---------------------------------------- + + def n(self) -> int: + return len(self.demonstrations) + + def demonstrators(self) -> Tuple[str, ...]: + out: List[str] = [] + for d in self.demonstrations: + if d.by not in out: + out.append(d.by) + return tuple(out) + + def conditions(self) -> Tuple[str, ...]: + out: List[str] = [] + for d in self.demonstrations: + if d.conditions not in out: + out.append(d.conditions) + return tuple(out) + + def center(self) -> Optional[float]: + """The mean of what was demonstrated, or None when nothing was. + + Defined at n=1, where it is simply the one value. That is the honest reading of a single + demonstration: a value was established, and nothing about a tolerance was. + """ + if not self.demonstrations: + return None + return sum(d.value for d in self.demonstrations) / len(self.demonstrations) + + def tolerance(self) -> Optional[Tuple[float, float]]: + """The demonstrated range, or None when there are too few demonstrations to state one. + + min and max of the observations, with no widening and no distribution assumed. The + interval is then a record of what happened rather than a model of what might, and it is + wrong only in the direction of being too narrow -- which is the safe direction, because + a narrow envelope asks for more demonstrations and a wide one hides a failure. + """ + if len(self.demonstrations) < MIN_DEMONSTRATIONS: + return None + values = [d.value for d in self.demonstrations] + return (min(values), max(values)) + + def spread(self) -> Optional[float]: + band = self.tolerance() + return None if band is None else band[1] - band[0] + + def refusal(self) -> Optional[str]: + """Why this envelope states no tolerance, or None when it states one. + + Reported rather than silently returning an interval anyway. A refusal that nobody can + read is indistinguishable from an answer nobody checked. + """ + if self.tolerance() is not None: + return None + n = self.n() + if n == 0: + return ( + f"'{self.operation}' has no recorded demonstration; there is no value and no " + "tolerance, and nothing here says what performing it well would look like" + ) + return ( + f"'{self.operation}' has {n} demonstration(s) of {self.metric}; a tolerance needs " + f"{MIN_DEMONSTRATIONS}. The value is established and the spread is not, so no envelope " + f"is stated. {MIN_DEMONSTRATIONS - n} more demonstration(s) would produce one" + ) + + def caveat(self) -> Optional[str]: + """What the envelope does not cover even where it states a tolerance. + + A range from one demonstrator describes that demonstrator. It is still the best evidence + available and it is not evidence about the operation, so the caveat travels with every + parity claim made against it rather than living in a methods section. + """ + if self.tolerance() is None: + return None + if len(self.demonstrators()) == 1: + return ( + f"every demonstration is by {self.demonstrators()[0]}; the range describes one " + "person's performance, not the spread of the operation" + ) + return None + + def describe(self) -> str: + band = self.tolerance() + if band is None: + return f"no envelope for {self.operation}" + return f"{self.metric} in {band[0]}-{band[1]} {self.units} over {self.n()} demonstrations" + + +@dataclass(frozen=True) +class Parity: + """A machine's performance placed against an expert envelope, with the reason it placed there. + + `expert_n` and `machine_n` are carried because the attainment is meaningless without them + and they are the first thing dropped when a result is summarized. MEETS over three and + three is a finding; MEETS over three and one is a bug this module refuses to produce. + """ + + operation: str + attainment: Attainment + reason: str + expert_n: int + machine_n: int + + @property + def demonstrated(self) -> bool: + return self.attainment.demonstrated_parity + + +def attainment( + envelope: Optional[Envelope], + observations: Sequence[MachineObservation], + operation: Optional[str] = None, +) -> Parity: + """Place machine observations against an expert envelope. + + The order of the checks is the substance of the function, because every one of them is a + path by which a comparison with too little behind it becomes a MEETS: + + no envelope, or no demonstrations nothing to attain; UNMEASURED + no machine observations nothing to place; UNMEASURED + none under the demonstrated conditions measured, but not of the same experiment + envelope states no tolerance a value with no spread cannot contain anything + too few machine observations a number that cannot be told from luck + otherwise compare the machine's WORST run to the envelope + + Conditions are matched by exact string equality, which is crude on purpose. The + alternative to a blunt match is no match, and no match compares a picogram run against a + nanogram demonstration and reports parity. + """ + # The untaught case -- no envelope AND no observations -- is the common one, and it is + # exactly the case where neither argument carries the operation's name. Without the + # caller supplying it the refusal reads "no expert has demonstrated 'unknown'", which + # names nothing and is useless as a queue entry. + op = ( + envelope.operation + if envelope is not None + else (observations[0].operation if observations else (operation or "unknown")) + ) + + if envelope is None or envelope.n() == 0: + return Parity( + operation=op, + attainment=Attainment.UNMEASURED, + reason=( + f"no expert has demonstrated '{op}'; there is no envelope to place a machine " + "against, and a machine number with nothing to compare it to is not a result" + ), + expert_n=0 if envelope is None else envelope.n(), + machine_n=len(observations), + ) + + if not observations: + return Parity( + operation=op, + attainment=Attainment.UNMEASURED, + reason=( + f"the expert side has {envelope.n()} demonstration(s) and the machine has never " + f"been measured on '{op}'; nothing is being compared" + ), + expert_n=envelope.n(), + machine_n=0, + ) + + demonstrated_conditions = envelope.conditions() + matching = tuple(o for o in observations if o.conditions in demonstrated_conditions) + if not matching: + return Parity( + operation=op, + attainment=Attainment.UNMEASURED, + reason=( + f"{len(observations)} machine observation(s) exist, none under the conditions the " + f"expert demonstrated ({'; '.join(demonstrated_conditions)}); under those conditions " + "the machine is unmeasured" + ), + expert_n=envelope.n(), + machine_n=0, + ) + + band = envelope.tolerance() + if band is None: + return Parity( + operation=op, + attainment=Attainment.INDISTINGUISHABLE_FROM_UNMEASURED, + reason=( + f"the machine has {len(matching)} matching observation(s) and the expert side states " + f"no tolerance: {envelope.refusal()}" + ), + expert_n=envelope.n(), + machine_n=len(matching), + ) + + if len(matching) < MIN_DEMONSTRATIONS: + return Parity( + operation=op, + attainment=Attainment.INDISTINGUISHABLE_FROM_UNMEASURED, + reason=( + f"{len(matching)} machine observation(s) of '{op}' against an envelope of " + f"{envelope.n()}; {MIN_DEMONSTRATIONS} are needed before the machine can be placed. " + "A run that landed inside the range cannot be told from one that got lucky" + ), + expert_n=envelope.n(), + machine_n=len(matching), + ) + + values = [o.value for o in matching] + low, high = min(values), max(values) + lo, hi = band + if envelope.goal is Goal.HIGHER: + meets = low >= lo + worst = f"worst run {low} {envelope.units} against a demonstrated floor of {lo}" + elif envelope.goal is Goal.LOWER: + meets = high <= hi + worst = f"worst run {high} {envelope.units} against a demonstrated ceiling of {hi}" + else: + meets = low >= lo and high <= hi + worst = f"runs span {low}-{high} {envelope.units} against a demonstrated {lo}-{hi}" + + caveat = envelope.caveat() + tail = f". Caveat: {caveat}" if caveat else "" + if meets: + return Parity( + operation=op, + attainment=Attainment.MEETS, + reason=( + f"{len(matching)} machine observation(s) attain the envelope from " + f"{envelope.n()} demonstration(s): {worst}{tail}" + ), + expert_n=envelope.n(), + machine_n=len(matching), + ) + return Parity( + operation=op, + attainment=Attainment.BELOW, + reason=( + f"{len(matching)} machine observation(s) do not attain the envelope from " + f"{envelope.n()} demonstration(s): {worst}{tail}" + ), + expert_n=envelope.n(), + machine_n=len(matching), + ) + + +# -- what would have to be demonstrated, per operation ------------------------- + + +@dataclass(frozen=True) +class Transferable: + """One operation whose expertise has to transfer, and the measurement that would decide it. + + A specification of a demonstration, not a demonstration -- the same kind of object as + `intelligence.Benchmark.how_to_measure` and `vision.VisualCheck.requires`. Writing it + before anyone performs it is what turns "we should watch someone do this" into an + experiment with a metric, a unit, and a direction, which is the difference between a + backlog and a wish. + + `why_this_metric` is prose and no checker reads it. It is here because the wrong metric is + the failure that survives every other check in this module: a demonstration measured on + something that does not decide the outcome produces a real envelope, a real attainment, + and a machine certified at performing the wrong thing well. + """ + + op: str + metric: str + units: str + goal: Goal + conditions: str # what a demonstration must hold constant for it to transfer + demonstrator: str # the kind of expert whose performance is the standard + why_this_metric: str + note: str = "" + + +# The operations in the reference protocols where an expert's performance is the standard +# and no protocol text captures it. Deliberately not every step: a step whose outcome is set +# by the instrument rather than by the person -- a thermal hold, a run-folder read -- has +# nothing to transfer, and listing it would inflate the backlog with work no demonstration +# would settle. +TRANSFERABLE: Tuple[Transferable, ...] = ( + Transferable( + op="start_sort", + metric="fraction of target wells containing exactly one cell", + units="fraction", + goal=Goal.HIGHER, + conditions="a known cell line at the working suspension density, 96-well plate, one cell per well", + demonstrator="a scientist who runs the sorter routinely and adjusts the gate by eye", + why_this_metric=( + "occupancy is what the rest of the run is spent on. Everything downstream costs a " + "reagent set per well whether or not a cell landed in it, so the operation's value is " + "the fraction of wells that hold one, and nothing else about the sort is worth " + "demonstrating first" + ), + note=( + "the expert contribution is the gate, which is set by looking at a scatter plot and " + "deciding where the population ends. Two scientists set it differently and both are " + "defensible, which is precisely why the spread across demonstrators is the quantity " + "worth having" + ), + ), + Transferable( + op="wgs_prep_lysis", + metric="delivered volume per well against nominal at the working volume", + units="uL", + goal=Goal.WINDOW, + conditions="the working volume of this prep, the tips actually used, a full plate", + demonstrator="a scientist pipetting the same addition by hand", + why_this_metric=( + "at picogram input a volume error is an efficiency change rather than a rounding, and " + "it biases the dataset instead of breaking it. Two sided because delivering more than " + "nominal is not better -- it changes the reaction the same way, in the other direction" + ), + note=( + "this is the operation where the machine most plausibly beats the person, and the " + "envelope is what would show it rather than assert it" + ), + ), + Transferable( + op="pcr_enrichment_round1_cleanup", + metric="fraction of input recovered through the full bead protocol", + units="fraction", + goal=Goal.HIGHER, + conditions="a spike-in of known quantity, the same bead ratio, the same elution volume", + demonstrator="a scientist who has done this cleanup enough to read a pellet", + why_this_metric=( + "recovery is the only quantity that distinguishes a cleanup that worked from one that " + "removed the library along with the supernatant, and both produce a plate that looks " + "identical and runs normally through every subsequent step" + ), + note=( + "the operation this repo's tacit layer already has a judgment about: a diffuse pellet " + "means rebind. A demonstration is what turns that judgment into a number a machine can " + "be held to" + ), + ), + Transferable( + op="library_pool", + metric="coefficient of variation of per-well representation in the pool", + units="percent", + goal=Goal.LOWER, + conditions="the same number of wells, normalized from the same quantification method", + demonstrator="a scientist normalizing and pooling by hand from a quant plate", + why_this_metric=( + "pooling is where a plate stops being 96 samples and becomes one tube. Uneven " + "representation is not recoverable afterward at any read depth, and it is invisible " + "until the run is already sequenced" + ), + ), + Transferable( + op="pcr_enrichment_round1", + metric="volume retained per well through the thermal hold", + units="fraction of starting volume", + goal=Goal.HIGHER, + conditions="the same seal, the same block, the same program, door closed", + demonstrator="a scientist sealing and loading the plate", + why_this_metric=( + "the thermal program belongs to the instrument; what a person contributes at this step " + "is the seal, and evaporation is what a bad one costs. Measuring block uniformity here " + "would measure the machine, which is a different question and already a benchmark" + ), + note=( + "this repo's own registry records that the choreography never closes the door around " + "the thermal leg, so a demonstration and a machine run today are not the same " + "configuration and `conditions` would not match" + ), + ), + Transferable( + op="run_program", + metric="delivered volume across the dilution series against nominal", + units="uL", + goal=Goal.WINDOW, + conditions="the same dilution factor, the same plate, the same tip type", + demonstrator="a scientist performing the serial dilution by hand", + why_this_metric=( + "a serial dilution compounds its own error: a systematic offset at the first transfer " + "is present in every subsequent one, so the metric has to be the delivered volume " + "rather than the final concentration, which hides the compounding" + ), + ), +) + + +TRANSFERABLE_BY_OP: Dict[str, Transferable] = {t.op: t for t in TRANSFERABLE} + + +# Every expert demonstration recorded in this repo. There are none. +# +# This is the finding, not a placeholder. Six instruments have been reverse-engineered, two +# protocols written out step by step, gates specified, failure modes enumerated -- and not +# one person has been measured performing any of it. Filling this tuple with a plausible +# series would make every function below return numbers, and every one of those numbers +# would be invented, which is the exact failure this module was written to refuse. The empty +# state is what makes `demonstration_queue` say something true. +DEMONSTRATIONS: Tuple[Demonstration, ...] = () + + +# Machine observations of the same metrics. Also none, and for a more interesting reason: +# the one mechanism this lab has timed is recorded in `throughput.DURATIONS` as a value and +# a worst case, which is a summary rather than a series. A summary has no n, so it cannot be +# placed against an envelope at all. Restating those seconds here would give this module a +# second copy of a fact `throughput` owns, and two copies of one number eventually disagree +# invisibly. +MACHINE_OBSERVATIONS: Tuple[MachineObservation, ...] = () + + +def _build_envelopes() -> Dict[str, Envelope]: + by_op: Dict[str, List[Demonstration]] = {} + for d in DEMONSTRATIONS: + by_op.setdefault(d.operation, []).append(d) + out: Dict[str, Envelope] = {} + for op, demos in by_op.items(): + spec = TRANSFERABLE_BY_OP.get(op) + if spec is None: + raise ValueError( + f"a demonstration of '{op}' is recorded, but no Transferable declares what it " + "measures; a demonstration whose metric nobody specified cannot be compared to " + "anything" + ) + out[op] = Envelope( + operation=op, + metric=spec.metric, + units=spec.units, + goal=spec.goal, + demonstrations=tuple(demos), + ) + return out + + +ENVELOPES: Dict[str, Envelope] = _build_envelopes() + + +def envelope_for(operation: str) -> Optional[Envelope]: + """The expert envelope for an operation, or None where nobody has demonstrated it.""" + return ENVELOPES.get(operation) + + +def observations_for(operation: str) -> Tuple[MachineObservation, ...]: + return tuple(o for o in MACHINE_OBSERVATIONS if o.operation == operation) + + +def demonstrations_still_needed(operation: str) -> int: + """How many more expert demonstrations before this operation has an envelope.""" + env = envelope_for(operation) + have = 0 if env is None else env.n() + return max(0, MIN_DEMONSTRATIONS - have) + + +def taught(operation: str) -> Tuple[bool, str]: + """Does an expert envelope exist for this operation at all? + + A missing demonstration is not a pass, in the same way a missing benchmark is not trust + and a missing measurement is not a met criterion. An operation nobody has demonstrated is + untaught, and an operation demonstrated once is still untaught: something was shown, and + what was shown carries no tolerance, so there is nothing for a machine to be held to. + """ + env = envelope_for(operation) + spec = TRANSFERABLE_BY_OP.get(operation) + if env is None: + if spec is None: + return False, ( + f"no expert demonstration of '{operation}' is recorded and none is specified; " + "nothing here says what 'as good as the expert' would mean for it" + ) + return False, ( + f"'{operation}' has a specified demonstration ({spec.metric}) and zero recorded; an " + f"SME has to perform it {MIN_DEMONSTRATIONS} times before there is an envelope" + ) + if env.tolerance() is None: + return False, env.refusal() or "no tolerance" + return True, ( + f"{env.n()} demonstration(s) from {len(env.demonstrators())} demonstrator(s): " + f"{env.describe()}" + ) + + +def untaught_operations(protocol) -> List[Tuple[str, str]]: + """(op, reason) for every distinct operation in a protocol with no expert envelope. + + Deliberately overlaps `intelligence.untrusted_ops` and answers a different question. That + one asks whether the machine has earned the right to run unattended; this one asks whether + anybody has established what running it well looks like. An operation can fail both, and + the work that fixes them is done by different people -- one is bench time on the + instrument, the other is bench time by the scientist, and the second cannot be bought. + """ + out: List[Tuple[str, str]] = [] + seen = set() + for step in protocol.steps: + if step.op in seen: + continue + seen.add(step.op) + ok, why = taught(step.op) + if not ok: + out.append((step.op, why)) + return out + + +# -- the transfer report ------------------------------------------------------- + + +@dataclass(frozen=True) +class TransferRow: + """One operation: what the expert established, what the machine did, and whether it attains. + + `benchmarks` is resolved from `intelligence` rather than restated, so this report cannot + disagree with that layer about what was asserted. The row carries both because the whole + point is that they are different evidence: side by side, an operation with a benchmark and + no envelope is visibly a target nobody has shown is reachable. + """ + + operation: str + envelope: Optional[Envelope] + observations: Tuple[MachineObservation, ...] + parity: Parity + benchmarks: Tuple[Benchmark, ...] + specified: bool # a Transferable declares what a demonstration would measure + + @property + def has_expert_envelope(self) -> bool: + return self.envelope is not None and self.envelope.tolerance() is not None + + @property + def asserted_only(self) -> bool: + """A target somebody asserted, and nobody has demonstrated is reachable.""" + return bool(self.benchmarks) and not self.has_expert_envelope + + +@dataclass +class TransferReport: + """Every operation asked the transfer question at once.""" + + rows: List[TransferRow] + + def untaught(self) -> List[TransferRow]: + return [r for r in self.rows if not r.has_expert_envelope] + + def attained(self) -> List[TransferRow]: + return [r for r in self.rows if r.parity.demonstrated] + + def asserted_only(self) -> List[TransferRow]: + """Operations with a benchmark and no envelope. + + The most common state in a real programme and the one that reads as progress: a number + exists, it is written down, it has a unit, and nobody has shown a person can hit it. + """ + return [r for r in self.rows if r.asserted_only] + + def unplaceable(self) -> List[TransferRow]: + """Operations where numbers exist on both sides and still say nothing.""" + return [ + r + for r in self.rows + if r.parity.attainment is Attainment.INDISTINGUISHABLE_FROM_UNMEASURED + ] + + def transfers(self) -> bool: + """True only if every operation in this report has an envelope the machine attains. + + The `bool(self.rows)` guard is the point of the method. An empty report satisfies `all()` + vacuously, and a transfer report over zero operations is not a lab that has finished + teaching its machines -- it is a report nobody gave any operations to. + """ + return bool(self.rows) and all(r.parity.demonstrated for r in self.rows) + + def counts(self) -> Dict[str, int]: + return { + "operations": len(self.rows), + "specified": sum(1 for r in self.rows if r.specified), + "with_envelope": sum(1 for r in self.rows if r.has_expert_envelope), + "attained": len(self.attained()), + "unplaceable": len(self.unplaceable()), + "asserted_only": len(self.asserted_only()), + } + + +def transfer_report(operations: Sequence[str]) -> TransferReport: + """For each operation: is there an envelope, is there machine performance, does it attain? + + Duplicate operations collapse, keeping the given order. Everything on a row is resolved + from the layer that owns it -- envelopes from the demonstrations, benchmarks from + `intelligence` -- so a row cannot state a target this package states differently elsewhere. + """ + rows: List[TransferRow] = [] + seen = set() + for op in operations: + if op in seen: + continue + seen.add(op) + env = envelope_for(op) + obs = observations_for(op) + rows.append( + TransferRow( + operation=op, + envelope=env, + observations=obs, + parity=attainment(env, obs, operation=op), + benchmarks=tuple(BENCHMARKS_BY_OP.get(op, ())), + specified=op in TRANSFERABLE_BY_OP, + ) + ) + return TransferReport(rows=rows) + + +# -- what an expert should demonstrate next ------------------------------------ + + +@dataclass(frozen=True) +class NextDemonstration: + """One entry in the SME queue: sit at this instrument, demonstrate these operations.""" + + instrument: str + operations: Tuple[str, ...] + demonstrations_needed: int + steps_blocked: int + + @property + def cost(self) -> int: + """Demonstrations to perform. A crude proxy for the expert's time, and an honest one: + each is one full run of the operation on real material, with the outcome measured.""" + return self.demonstrations_needed + + +def demonstration_queue(protocols: Sequence) -> List[NextDemonstration]: + """The SME queue, ranked by how many untaught operations one sitting would settle. + + Grouped by INSTRUMENT for the same reason `ledger.unlocks` is, and the reason is not the + same strength of constraint, which is worth being exact about. The ledger's grouping is + forced by code: the coverage gate refuses an armed run while any sibling command is + undecoded, so a single decode unblocks exactly zero steps. This grouping is forced by + physics and scheduling instead -- a demonstration needs the expert, the instrument, the + labware, and real material in one place at one time, so the unit of progress is a sitting + rather than an operation. Ranking individual operations would produce a queue whose top + three entries are three separate afternoons of the same person's time. + + Scoped to the operations the given protocols actually contain, and the ranking is computed + from them: hand it a different set of protocols and the order changes, because a station + carrying operations in several flows is where a scientist's afternoon buys the most. + """ + ops_by_instrument: Dict[str, List[str]] = {} + steps: Dict[str, int] = {} + needed: Dict[str, int] = {} + + for protocol in protocols: + for step in protocol.steps: + if step.op not in TRANSFERABLE_BY_OP: + continue + ok, _why = taught(step.op) + if ok: + continue + ops = ops_by_instrument.setdefault(step.instrument, []) + if step.op not in ops: + ops.append(step.op) + needed[step.instrument] = needed.get(step.instrument, 0) + demonstrations_still_needed( + step.op + ) + steps[step.instrument] = steps.get(step.instrument, 0) + 1 + + out = [ + NextDemonstration( + instrument=key, + operations=tuple(ops), + demonstrations_needed=needed.get(key, 0), + steps_blocked=steps.get(key, 0), + ) + for key, ops in ops_by_instrument.items() + ] + out.sort(key=lambda q: (-len(q.operations), -q.steps_blocked, q.cost, q.instrument)) + return out + + +def teaching_summary() -> Dict[str, int]: + """How much of the expert performance behind these protocols has been recorded.""" + return { + "operations_specified": len(TRANSFERABLE), + "demonstrations_recorded": len(DEMONSTRATIONS), + "operations_with_envelope": sum( + 1 for op in TRANSFERABLE_BY_OP if taught(op)[0] + ), + "machine_observations_recorded": len(MACHINE_OBSERVATIONS), + "minimum_demonstrations": MIN_DEMONSTRATIONS, + } + + +__all__ = [ + "DEMONSTRATIONS", + "ENVELOPES", + "MACHINE_OBSERVATIONS", + "MIN_DEMONSTRATIONS", + "TRANSFERABLE", + "TRANSFERABLE_BY_OP", + "Attainment", + "Demonstration", + "Envelope", + "Goal", + "MachineObservation", + "NextDemonstration", + "Parity", + "Transferable", + "TransferReport", + "TransferRow", + "attainment", + "demonstration_queue", + "demonstrations_still_needed", + "envelope_for", + "observations_for", + "taught", + "teaching_summary", + "transfer_report", + "untaught_operations", +] diff --git a/tests/test_coverage.py b/tests/test_coverage.py new file mode 100644 index 0000000..1402315 --- /dev/null +++ b/tests/test_coverage.py @@ -0,0 +1,495 @@ +"""Device-free tests for the composition of vision, QC gates, and the failure model. + +Every test here is an attempt to make this lab look covered when nothing would catch +anything. The composition is the most dangerous layer in the package for exactly that +reason: it is the one a reader trusts, it prints a table with a covered column, and both of +its inputs offer a plausible way to fill that column with something that does not exist. A +visual check is declared for a failure -- count it. A gate is declared for a failure -- +count it. Do that and a workcell with no camera and a plate reader that has never returned a +number reports full coverage while its material is being destroyed silently. + +So the tests that matter are the ones that hand the module something that exists and is not +usable, and insist it says so: + + a declared visual check with no camera behind it + a gate that cannot be evaluated because its input comes from a broken instrument + a gate that CAN be evaluated and reads the wrong assay for the sample + a report over no rows at all, which is the vacuous pass in its purest form + a vision capability with every requirement met, which must STILL leave the INVISIBLE + failures uncovered, because that boundary is physics and not budget + +and the one on the other side, which is the reason the module is worth having: a +material-destroying failure with neither path must appear in the report every time, never +omitted and never softened. +""" + +from __future__ import annotations + +import dataclasses + +import pytest + +from autonomous_lab import build_ledger, protocols +from autonomous_lab.coverage import ( + Cover, + CoverageReport, + CoverageRow, + Uncovered, + coverage_report, + mandatory_gates, + sota_lift, + unmet_demands, +) +from autonomous_lab.qc import ( + LIBRARY_QUANT_GATE, + SORT_OCCUPANCY_GATE, + GateReadiness, + GateReport, + Readiness, + gate_report, +) +from autonomous_lab.recovery import FAILURES_BY_NAME, Detection, Severity, recovery_report +from autonomous_lab.vision import ( + CHECKS_BY_NAME, + Observable, + VisionCapability, + VisionRequirement, + check_catching, +) + + +def _fully_equipped() -> VisionCapability: + """A workcell holding every vision requirement. No such bench exists in this repo.""" + return VisionCapability(name="hypothetical", satisfied=tuple(VisionRequirement)) + + +def _genomics(): + protocol = protocols.get("single_cell_genomics") + ledger = build_ledger(protocol) + return protocol, ledger, gate_report(protocol.name, ledger) + + +def _report(capability=None, gates=None) -> CoverageReport: + """The composition over the genomics loop, at a stated vision capability. + + The recovery report is rebuilt against the same capability every time. That is not + ceremony: coverage_report refuses a recovery report built against a different one, and it + refuses it because a mismatch produces rows that are individually well-formed and describe + a workcell nobody has. + """ + cap = capability or VisionCapability.none() + protocol, ledger, real_gates = _genomics() + gates = real_gates if gates is None else gates + return coverage_report( + protocol, ledger, gates, recovery_report(protocol, gates, cap), cap + ) + + +def _row(report: CoverageReport, name: str) -> CoverageRow: + return next(r for r in report.rows if r.failure.name == name) + + +def _gates_where(*readiness_rows) -> GateReport: + """A gate report asserting a readiness the real ledger does not support. + + Built by hand on purpose. Every gate in this repo is unsatisfiable, so the only way to + test that an EVALUABLE gate is treated correctly -- both when it counts and when it does + not -- is to hand the module a workcell that does not exist. + """ + return GateReport(protocol="single_cell_genomics", rows=list(readiness_rows)) + + +# -- the vacuous pass, in each of its costumes --------------------------------- + + +def test_a_declared_visual_check_that_is_not_deployable_is_not_coverage(): + """The failure this module exists to refuse: counting a detector nobody has built. + + tips_present_after_pickup is declared, names the failure it catches, and is the easiest + check in the list. None of that puts a camera on the instrument. + """ + report = _report() + row = _row(report, "tip_not_picked_up") + + assert row.vision_check == "tips_present_after_pickup" + assert not row.vision_deployable + assert row.cover is Cover.NEITHER + assert VisionRequirement.CAMERA.value in row.vision_reason + + +def test_a_camera_alone_is_not_coverage(): + """The requirement labs stall on is validation, and an unvalidated detector is not one.""" + camera_only = VisionCapability(name="camera only", satisfied=(VisionRequirement.CAMERA,)) + report = _report(camera_only) + for row in report.rows: + assert not row.vision_deployable, f"{row.failure.name} counted a check with no validation" + assert report.counts()["covered"] == 0 + + +def test_an_unsatisfiable_gate_is_not_coverage(): + """A gate whose input comes from an instrument that has never returned a number. + + bead_pellet_aspirated declares the library-quant gate. The gate is real, it is on the + protocol, and the Tecan times out deterministically, so it can never fire. Counting it + would report the most destructive routine failure in the loop as handled. + """ + report = _report() + row = _row(report, "bead_pellet_aspirated") + + assert row.gate == "library_quant_before_flow_cell" + assert not row.gate_evaluable + assert "unsatisfiable" in row.gate_reason + assert row.cover is Cover.NEITHER + assert row.destroys_material + + +def test_an_evaluable_gate_resting_on_the_wrong_assay_is_not_coverage(): + """The quietest costume: the gate fires, the number is real, and it is about something else. + + A260 at picogram input counts carrier, primer, and free nucleotide. Repair the plate + reader and this gate becomes evaluable and still passes an empty library, so evaluability + alone must not be allowed to fill the covered column. + """ + repaired = _gates_where( + GateReadiness(LIBRARY_QUANT_GATE, Readiness.READY, "pretend the reader was fixed today") + ) + report = _report(gates=repaired) + row = _row(report, "enzyme_activity_lost") + + assert row.gate == "library_quant_before_flow_cell" + assert not row.gate_evaluable + assert "wrong assay" in row.gate_reason + assert row.cover is Cover.NEITHER + + +def test_an_empty_report_is_not_complete(): + """A composition handed no rows must not report an all-clear. + + The same bug as a gate evaluated against an empty measurement dict. Every criterion is + met, because there were none. + """ + empty = CoverageReport(protocol="nothing", capability=VisionCapability.none(), rows=[]) + assert not empty.complete() + assert empty.counts()["total"] == 0 + assert empty.uncovered_destructive() == [] + + +def test_mandatory_gates_with_no_gate_report_is_a_skip_and_not_a_pass(): + """A default that reads as a PASS when it is really a SKIP would let a caller comply by + supplying nothing at all.""" + demands = mandatory_gates() + assert demands + for demand in demands: + assert not demand.met + assert "unresolved" in demand.reason + assert len(unmet_demands()) == len(demands) + + +# -- the ceiling on vision, held through the composition ----------------------- + + +def test_full_vision_equipment_still_leaves_destructive_failures_uncovered(): + """The honest bound, restated over the composition rather than over vision alone. + + If this ever passes trivially, someone has reclassified an INVISIBLE failure or started + counting a gate that cannot fire. + """ + report = _report(_fully_equipped()) + uncovered = report.uncovered_destructive() + assert uncovered, "fully equipped, and the invisible failures still destroy material unseen" + assert not report.complete() + + names = {row.failure.name for row in uncovered} + assert "enzyme_activity_lost" in names + assert "cross_contamination" in names + assert "volume_short_within_tolerance" in names + + +def test_no_capability_makes_an_invisible_failure_vision_covered(): + """The boundary is physics. Every capability in the test suite has to respect it.""" + for capability in (VisionCapability.none(), _fully_equipped()): + for row in _report(capability).rows: + if row.observable is Observable.INVISIBLE: + assert not row.vision_deployable, ( + f"{row.failure.name} is INVISIBLE and was reported as covered by a camera" + ) + assert row.cover is not Cover.VISION + + +def test_structural_and_fixable_gaps_are_not_confused(): + """Two empty coverage cells that print alike and imply different budgets.""" + report = _report() + + assert _row(report, "tip_not_picked_up").uncovered is Uncovered.FIXABLE + assert _row(report, "volume_short_within_tolerance").uncovered is Uncovered.STRUCTURAL + assert Uncovered.STRUCTURAL.needs_a_new_instrument + assert not Uncovered.FIXABLE.needs_a_new_instrument + + # A gate that exists and is merely dead is an instrument repair, not a re-instrumentation. + assert _row(report, "enzyme_activity_lost").uncovered is Uncovered.FIXABLE + for row in report.structural(): + assert row.observable is Observable.INVISIBLE + assert row.gate is None + + +# -- coverage the sibling layers cannot report --------------------------------- + + +def test_the_composition_finds_coverage_recovery_alone_would_miss(): + """The reason the module exists rather than being a view over recovery. + + bead_pellet_aspirated declares a gate, so recovery consults vision for it never. A + declared visual check watches exactly that condition, and with a camera in place it holds + the failure whether or not the plate reader is ever repaired. + """ + protocol, ledger, gates = _genomics() + equipped = _fully_equipped() + silent = { + row.failure.name for row in recovery_report(protocol, gates, equipped).destructive_and_silent() + } + assert "bead_pellet_aspirated" in silent + + row = _row(_report(equipped), "bead_pellet_aspirated") + assert row.cover is Cover.VISION + assert not row.gate_evaluable + + +def test_an_evaluable_gate_on_an_appropriate_assay_is_coverage(): + """The positive control. Without it, every refusal above could be passing for free.""" + sorted_ok = _gates_where( + GateReadiness(SORT_OCCUPANCY_GATE, Readiness.READY, "pretend the sorter were decoded") + ) + row = _row(_report(gates=sorted_ok), "empty_well_after_sort") + assert row.gate_evaluable + assert row.cover is Cover.GATE + assert row.uncovered is None + assert "already held" in row.closes_it() + + +def test_both_instruments_are_reported_as_both(): + """Not redundancy for its own sake: it is the only state that survives one going down.""" + sorted_ok = _gates_where( + GateReadiness(SORT_OCCUPANCY_GATE, Readiness.READY, "pretend the sorter were decoded") + ) + row = _row(_report(_fully_equipped(), gates=sorted_ok), "empty_well_after_sort") + assert row.vision_deployable and row.gate_evaluable + assert row.cover is Cover.BOTH + assert Cover.BOTH.covered + + +# -- a destructive failure with neither path is never omitted ------------------ + + +def test_every_destructive_failure_with_neither_path_is_reported(): + """The one omission that would make the whole report worse than useless.""" + report = _report() + reported = {row.failure.name for row in report.uncovered_destructive()} + for row in report.rows: + if not row.cover.covered and row.destroys_material: + assert row.failure.name in reported, f"{row.failure.name} was computed and then dropped" + + assert { + "bead_pellet_aspirated", + "cross_contamination", + "index_misassignment", + "odtc_callback_theft", + "enzyme_activity_lost", + "volume_short_within_tolerance", + "wrong_reagent_right_position", + } <= reported + + +def test_the_genomics_loop_covers_nothing_today(): + """Pinned, because it is the finding. Zero of fourteen failures have a machine check.""" + report = _report() + counts = report.counts() + assert counts["covered"] == 0 + assert counts["uncovered"] == counts["total"] + assert counts["uncovered_destructive"] > 0 + assert not report.complete() + + +def test_an_uncovered_row_never_denies_a_path_recovery_already_found(): + """Uncovered means no camera and no gate. It must not be read as nothing anywhere. + + The STAR raises USBError before anything moves, and demanding a camera for that would be + noise dressed as rigor. + """ + report = _report() + loud = _row(report, "star_resource_busy") + assert loud.cover is Cover.NEITHER + assert loud.residual is Detection.INSTRUMENT + assert not loud.unwatched + assert "instrument" in loud.closes_it() + + late = _row(report, "cross_contamination") + assert late.residual is Detection.DATA_ANALYSIS + assert "after the flow cell is spent" in late.closes_it() + + gone = _row(report, "odtc_callback_theft") + assert gone.residual is None + assert gone.unwatched + + +def test_every_uncovered_row_names_what_would_close_it(): + """A remedy column that reads 'improve coverage' has handed the problem back.""" + for capability in (VisionCapability.none(), _fully_equipped()): + for row in _report(capability).uncovered(): + remedy = row.closes_it() + assert len(remedy) > 40 + assert "this module cannot name" not in remedy + + +# -- sota_lift: what a better model buys, and what it cannot ------------------ + + +def test_sota_lift_never_reports_a_lift_for_an_invisible_failure(): + """The overclaim the function exists to prevent, asserted directly.""" + protocol, ledger, gates = _genomics() + lift = sota_lift(VisionCapability.none(), _fully_equipped(), protocol, ledger, gates) + + assert lift.lifted + for moved in lift.lifted: + assert moved.failure.observable.reachable_by_vision + assert moved.failure.observable is not Observable.INVISIBLE + assert moved.before is Cover.NEITHER + assert moved.after.covered + assert CHECKS_BY_NAME[moved.check].catches == moved.failure.name + + +def test_sota_lift_reports_the_failures_no_capability_moves(): + """The second list is the product. A lift shown without it is a purchase justification.""" + protocol, ledger, gates = _genomics() + lift = sota_lift(VisionCapability.none(), _fully_equipped(), protocol, ledger, gates) + + ceiling = {c.failure.name for c in lift.ceiling} + for name in ("enzyme_activity_lost", "cross_contamination", "index_misassignment"): + assert name in ceiling + assert FAILURES_BY_NAME[name].observable is Observable.INVISIBLE + assert lift.destructive_ceiling(), "the ceiling holds failures that destroy material" + assert set(ceiling).isdisjoint({m.failure.name for m in lift.lifted}) + + +def test_sota_lift_at_the_top_of_the_ladder_still_has_a_ceiling(): + """Proposing the best possible capability does not empty the ceiling, at any price.""" + protocol, ledger, gates = _genomics() + lift = sota_lift(_fully_equipped(), _fully_equipped(), protocol, ledger, gates) + assert lift.buys_nothing, "a capability proposed against itself cannot buy anything" + assert lift.destructive_ceiling() + + +def test_a_lift_names_the_requirement_the_upgrade_actually_supplied(): + """A lift that cannot say what changed is a coincidence, not a purchase argument.""" + protocol, ledger, gates = _genomics() + lift = sota_lift(VisionCapability.none(), _fully_equipped(), protocol, ledger, gates) + for moved in lift.lifted: + assert moved.gained + assert VisionRequirement.VALIDATION in moved.gained + + +# -- gates that are not optional ------------------------------------------------ + + +def test_every_invisible_destructive_failure_demands_a_gate(): + """Where no camera can substitute, an assay is not one option among several.""" + _, _, gates = _genomics() + demands = mandatory_gates(gates) + demanded = {d.failure.name for d in demands} + + for failure in FAILURES_BY_NAME.values(): + if failure.observable is Observable.INVISIBLE and failure.severity in ( + Severity.SAMPLE, + Severity.RUN, + ): + assert failure.name in demanded, f"{failure.name} destroys material unseen and demands nothing" + + for demand in demands: + assert demand.failure.observable is Observable.INVISIBLE + assert "no camera substitutes" in demand.statement() + + +def test_no_mandatory_gate_is_met_in_this_lab(): + """Pinned. Six gates must exist and none of them can fire today.""" + _, _, gates = _genomics() + demands = mandatory_gates(gates) + assert demands + assert all(not d.met for d in demands) + assert len(unmet_demands(gates)) == len(demands) + + named_nothing = [d for d in demands if d.gate is None] + assert named_nothing, "most of these failures have no assay named for them at all" + + +# -- the module refuses inputs it cannot compose -------------------------------- + + +def test_composing_reports_about_different_protocols_is_refused(): + """A blend of two labs is a number about no lab.""" + genomics, ledger, gates = _genomics() + chemistry = protocols.get("small_molecule_qc") + wrong = recovery_report(chemistry, None, VisionCapability.none()) + with pytest.raises(ValueError, match="one protocol"): + coverage_report(genomics, ledger, gates, wrong, VisionCapability.none()) + + +def test_a_recovery_report_built_against_another_capability_is_refused(): + """The quieter mismatch: every row is well-formed and describes a workcell nobody has.""" + protocol, ledger, gates = _genomics() + stale = recovery_report(protocol, gates, VisionCapability.none()) + with pytest.raises(ValueError, match="disagree"): + coverage_report(protocol, ledger, gates, stale, _fully_equipped()) + + +# -- the house rules the report itself has to obey ------------------------------ + + +def test_coverage_is_derived_and_never_a_stored_field(): + """An author-settable `cover` would be a claim of coverage, which is what this refuses. + + It is also the general rule: a derived value kept beside the values it derives from + eventually disagrees with them, and nothing reveals which one is wrong. + """ + stored = {f.name for f in dataclasses.fields(CoverageRow)} + for derived in ("cover", "uncovered", "severity", "observable", "destroys_material"): + assert derived not in stored, f"'{derived}' is derivable and must not be storable" + + +def test_a_reported_cover_is_backed_by_the_data_that_produced_it(): + """A claim of coverage the module's own row contradicts.""" + sorted_ok = _gates_where( + GateReadiness(SORT_OCCUPANCY_GATE, Readiness.READY, "pretend the sorter were decoded") + ) + equipped = _fully_equipped() + for row in _report(equipped, gates=sorted_ok).rows: + if row.cover in (Cover.VISION, Cover.BOTH): + check = check_catching(row.failure.name) + assert check is not None and equipped.available(check) + assert check.name == row.vision_check + if row.cover in (Cover.GATE, Cover.BOTH): + readiness = next(r for r in sorted_ok.rows if r.gate.name == row.gate) + assert readiness.evaluable + assert row.cover.covered == (row.uncovered is None) + assert row.cover.closed_by + + +def test_coverage_agrees_with_recovery_about_what_destroys_material(): + """Two definitions of one line eventually disagree, so there must only be one.""" + protocol, ledger, gates = _genomics() + capability = VisionCapability.none() + recovery = recovery_report(protocol, gates, capability) + report = coverage_report(protocol, ledger, gates, recovery, capability) + + destructive_here = {row.failure.name for row in report.rows if row.destroys_material} + destructive_there = {row.failure.name for row in recovery.destructive_and_silent()} + assert destructive_there <= destructive_here + + +def test_the_counts_add_up(): + """A report whose own arithmetic disagrees cannot be trusted about anything else.""" + for capability in (VisionCapability.none(), _fully_equipped()): + counts = _report(capability).counts() + assert counts["covered"] + counts["uncovered"] == counts["total"] + assert counts["vision"] + counts["gate"] + counts["both"] == counts["covered"] + assert counts["fixable"] + counts["structural"] == counts["uncovered"] + assert counts["unwatched"] <= counts["uncovered"] + assert counts["uncovered_destructive"] <= counts["uncovered"] diff --git a/tests/test_durability.py b/tests/test_durability.py new file mode 100644 index 0000000..24f9401 --- /dev/null +++ b/tests/test_durability.py @@ -0,0 +1,464 @@ +"""Device-free tests for the durability layer. + +The tests that matter here try to make an untracked instrument look healthy. Four ways in: +an instrument with no service history, an obligation charged in a unit nobody counted, a +maintenance record that software wrote to itself, and an obligation list that is empty +because nobody has written the maintenance plan yet. Every one of them is a SKIP that reads +as a PASS, and every one of them ends the same way -- a campaign starts on a machine nobody +could vouch for, and the run record is impeccable about everything except that. + +The fifth test hunts the number. This module must not return a predicted failure rate under +any name, because a reliability figure invented here would become a specification nobody +measured. +""" + +from __future__ import annotations + +import dataclasses +import inspect + +import pytest + +from autonomous_lab import Workcell +from autonomous_lab import durability +from autonomous_lab.durability import ( + NOT_COMPUTED, + OBLIGATIONS, + SERVICE_HISTORY, + Accrued, + Campaign, + Charge, + Entitlement, + InstrumentHealth, + Interval, + IntervalKind, + ServiceRecord, + crosses_boundary, + entitlement_summary, + health_for, + refusals, + standing, + untrusted_instruments, +) +from autonomous_lab.provenance import Attestation +from autonomous_lab.qc import Basis + + +def _calibration(**kwargs) -> Interval: + """A calibration obligation. Every number here is the test's, not the repo's.""" + fields = dict( + name="absorbance_two_point", + instrument="tecan", + kind=IntervalKind.CALIBRATION, + restores="run the two-point calibration and record both readings", + basis=Basis.VENDOR, + every_runs=100.0, + ) + fields.update(kwargs) + return Interval(**fields) + + +def _serviced(interval: Interval, since: Accrued, attestation=Attestation.WITNESSED) -> ServiceRecord: + return ServiceRecord( + interval=interval.name, + performed="day 0", + by="technician", + attestation=attestation, + since=since, + ) + + +def _health(interval: Interval, since=None, attestation=Attestation.WITNESSED) -> InstrumentHealth: + records = () if since is None else (_serviced(interval, since, attestation),) + return InstrumentHealth(instrument=interval.instrument, obligations=(interval,), records=records) + + +# -- an instrument nobody tracked is not a healthy instrument ------------------ + + +def test_an_instrument_with_no_service_history_is_not_healthy(): + """The single most important test in the file. + + The natural implementation asks "is anything past due", finds no record of anything + being due, and returns entitled. That is how a lab discovers in month three that the + pump it has been running for two years has never been opened. + """ + interval = _calibration() + health = _health(interval) + assert health.entitlement() is Entitlement.UNKNOWN + assert not health.entitlement().trustworthy + row = standing(health, interval) + assert row.entitlement is Entitlement.UNKNOWN + assert "no service record" in row.reason + + +def test_unknown_is_not_optimistic_and_ranks_below_lapsed(): + """Unmeasured is untrusted by default, and worse than known-overdue. + + A lapse names its own fix. An unrecorded obligation does not even establish what state + the instrument is in, so it needs the service AND the record-keeping. + """ + assert not Entitlement.UNKNOWN.trustworthy + assert not Entitlement.UNKNOWN.entitled_now + assert not Entitlement.LAPSED.trustworthy + assert Entitlement.ENTITLED.trustworthy + assert durability._ENTITLEMENT_ORDER.index(Entitlement.UNKNOWN) > durability._ENTITLEMENT_ORDER.index( + Entitlement.LAPSED + ) + + +def test_no_declared_obligations_is_not_a_clean_bill_of_health(): + """The vacuous pass, arriving through an empty list instead of an empty dict. + + `all()` over no obligations is True, so an instrument nobody wrote a maintenance plan + for would clear every check it was ever put through. + """ + health = InstrumentHealth(instrument="star") + assert health.entitlement() is Entitlement.UNKNOWN + report = crosses_boundary(health, Campaign(runs=500)) + assert report.rows == () + assert not report.clear + assert "has not been shown to need none" in report.why_not() + + +def test_an_asserted_service_record_does_not_discharge_the_obligation(): + """Software marking its own scheduled job complete is intent, not evidence. + + The same line `provenance.Attestation` draws between a command log and a record of + events, pointed at a maintenance system. + """ + interval = _calibration() + health = _health(interval, since=Accrued(runs=5), attestation=Attestation.ASSERTED) + assert health.entitlement() is Entitlement.UNKNOWN + assert "asserted" in standing(health, interval).reason + + witnessed = _health(interval, since=Accrued(runs=5), attestation=Attestation.WITNESSED) + assert witnessed.entitlement() is Entitlement.ENTITLED + + +def test_a_record_that_counts_nothing_does_not_restore_entitlement(): + """An obligation charged in runs, against a lab that never counted runs. + + There is no remaining budget here -- not a large one, none that can be stated. A number + returned over this input would be a number over an input nobody measured. + """ + interval = _calibration(every_runs=100.0) + health = _health(interval, since=Accrued(days=3.0)) # days counted, runs not + assert health.entitlement() is Entitlement.UNKNOWN + row = standing(health, interval) + assert Charge.RUNS in row.remaining.uncounted + assert row.remaining.value(Charge.RUNS) is None + assert not row.remaining.computable + + +def test_an_uncounted_unit_is_not_read_as_zero(): + """The default that would silently restore entitlement to every untracked instrument.""" + assert Accrued().charged(Charge.RUNS) is None + assert Accrued(runs=0).charged(Charge.RUNS) == 0 + interval = _calibration() + fresh = _health(interval, since=Accrued(runs=0)) + assert fresh.entitlement() is Entitlement.ENTITLED + untracked = _health(interval, since=Accrued()) + assert untracked.entitlement() is Entitlement.UNKNOWN + + +# -- lapse ---------------------------------------------------------------------- + + +def test_an_exhausted_budget_is_lapsed(): + interval = _calibration(every_runs=100.0) + health = _health(interval, since=Accrued(runs=100.0)) + assert health.entitlement() is Entitlement.LAPSED + assert standing(health, interval).remaining.exhausted() == (Charge.RUNS,) + + +def test_the_first_unit_to_run_out_governs(): + """An interval that recurs on days and runs is two obligations sharing a name.""" + interval = _calibration(every_runs=100.0, every_days=365.0) + health = _health(interval, since=Accrued(runs=2.0, days=400.0)) + assert health.entitlement() is Entitlement.LAPSED + row = standing(health, interval) + assert row.remaining.exhausted() == (Charge.DAYS,) + assert row.remaining.value(Charge.RUNS) == 98.0 + + +def test_calendar_time_lapses_an_instrument_that_did_no_work(): + """The wear model that only counts usage believes a mothballed instrument stays fresh.""" + assert Charge.DAYS.accrues_when_idle + assert not Charge.RUNS.accrues_when_idle + assert not Charge.HOURS.accrues_when_idle + + interval = _calibration(every_days=365.0, every_runs=None) + idle = _health(interval, since=Accrued(days=400.0, runs=0.0)) + assert idle.entitlement() is Entitlement.LAPSED + + +def test_a_calibration_lapse_reaches_backwards_and_a_tip_box_does_not(): + """The distinction that decides what a lapse costs. + + A depleted consumable stops the next run. An expired calibration means every number + since is unattributable, and no downstream analysis recovers that. + """ + assert IntervalKind.CALIBRATION.invalidates_past_results + assert IntervalKind.QUALIFICATION.invalidates_past_results + assert not IntervalKind.CONSUMABLE.invalidates_past_results + assert not IntervalKind.PREVENTIVE_SERVICE.invalidates_past_results + + interval = _calibration(every_runs=10.0) + health = _health(interval, since=Accrued(runs=50.0)) + assert health.invalidating() + assert "unattributable" in health.lapsed()[0].reason + + +# -- the boundary crossing ------------------------------------------------------ + + +def test_a_campaign_spanning_a_calibration_expiry_is_flagged_though_every_instrument_is_entitled(): + """The load-bearing case, and the reason a current-state check is not enough. + + Nothing is overdue. Every instrument passes a health check taken this morning. The + campaign still cannot run, because a plate started before the boundary and finished + after it has a provenance nothing downstream repairs. + """ + interval = _calibration(every_runs=100.0) + health = _health(interval, since=Accrued(runs=60.0)) + assert health.entitlement() is Entitlement.ENTITLED, "entitled today, by every current check" + + report = crosses_boundary(health, Campaign(runs=120)) + assert not report.clear + crossings = report.crossings() + assert len(crossings) == 1 + assert crossings[0].entitlement is Entitlement.EXPIRING + assert crossings[0].crossing == (Charge.RUNS,) + assert "before the campaign starts" in crossings[0].restores + assert health.entitlement(Campaign(runs=120)) is Entitlement.EXPIRING + + +def test_a_campaign_inside_the_remaining_budget_is_clear(): + interval = _calibration(every_runs=100.0) + health = _health(interval, since=Accrued(runs=60.0)) + report = crosses_boundary(health, Campaign(runs=39)) + assert report.clear + assert report.crossings() == () + + +def test_expiring_is_a_deadline_rather_than_a_defect(): + """EXPIRING must not read as unfit; the fix is to the plan, not to the machine.""" + assert Entitlement.EXPIRING.entitled_now + assert not Entitlement.EXPIRING.trustworthy + + +def test_calendar_days_are_charged_against_an_instrument_the_campaign_never_touches(): + """A ninety-day campaign charges ninety days to every instrument in the lab. + + A plan that counts only its own plates misses the boundary that catches it. + """ + interval = _calibration(every_days=365.0, every_runs=None) + health = _health(interval, since=Accrued(days=300.0)) + report = crosses_boundary(health, Campaign(runs=0, days=90)) + assert report.crossings(), "an idle instrument still crosses a calendar boundary" + assert report.crossings()[0].crossing == (Charge.DAYS,) + + +def test_an_uncounted_budget_cannot_be_shown_not_to_cross(): + """An unavailable answer is reported as one, not folded in with the real crossings.""" + interval = _calibration(every_runs=100.0) + health = _health(interval, since=Accrued(days=3.0)) + report = crosses_boundary(health, Campaign(runs=10)) + assert report.crossings() == () + assert report.uncertain() + assert not report.clear + + +def test_an_already_lapsed_instrument_does_not_report_a_clear_campaign(): + """Nothing crosses, because the boundary is already behind it. That is not clear.""" + interval = _calibration(every_runs=100.0) + health = _health(interval, since=Accrued(runs=140.0)) + report = crosses_boundary(health, Campaign(runs=5)) + assert report.crossings() == () + assert report.lapsed() + assert not report.clear + + +def test_a_campaign_that_plans_no_work_is_refused(): + """The emptiest pass available: asking whether zero plates cross a boundary.""" + interval = _calibration() + health = _health(interval, since=Accrued(runs=1.0)) + with pytest.raises(ValueError, match="crosses nothing"): + crosses_boundary(health, Campaign()) + + +def test_the_crossing_docstring_names_the_provenance_vocabulary(): + """The tie is load-bearing prose, so a rewrite that drops it should be loud.""" + doc = crosses_boundary.__doc__ or "" + assert "CONFIRMED" in doc + assert "provenance" in doc + + +# -- basis: a borrowed interval is not a validated one ------------------------- + + +def test_a_vendor_interval_is_not_validated_for_this_duty_cycle(): + """Real evidence, and not evidence about this workload on this input.""" + assert not _calibration(basis=Basis.VENDOR).validated + assert not _calibration(basis=Basis.LITERATURE).validated + assert not _calibration(basis=Basis.INTUITION).validated + assert _calibration(basis=Basis.IN_HOUSE).validated + + +def test_an_unvalidated_interval_still_governs_entitlement(): + """Basis records what the number is worth; it does not excuse the obligation. + + Ignoring a vendor interval because it came from a manual would be worse than honoring + it. The finding is that the number is borrowed, not that it can be skipped. + """ + interval = _calibration(basis=Basis.VENDOR, every_runs=100.0) + health = _health(interval, since=Accrued(runs=150.0)) + assert health.entitlement() is Entitlement.LAPSED + assert health.unvalidated() == (interval,) + + +def test_an_interval_that_recurs_on_nothing_is_refused(): + """An obligation that never comes due would make every instrument entitled forever.""" + with pytest.raises(ValueError, match="recurs on no unit"): + _calibration(every_runs=None, every_days=None, every_hours=None) + + +def test_a_nonpositive_interval_is_refused(): + with pytest.raises(ValueError, match="already past due"): + _calibration(every_runs=0.0) + + +# -- no field may silence the report ------------------------------------------- + + +def test_health_cannot_declare_itself_entitled(): + """There is no author-settable field that makes an instrument healthy. + + The same rule `qc` applies to gate readiness and the ledger applies to automation. If + this fails, someone added an override and the report can be made to lie. + """ + names = {f.name for f in dataclasses.fields(InstrumentHealth)} + for forbidden in ("entitled", "healthy", "ok", "waived", "override", "exempt", "status"): + assert forbidden not in names, f"InstrumentHealth carries a settable '{forbidden}'" + + +def test_entitlement_is_derived_on_demand_rather_than_stored(): + """A derived value kept beside its inputs eventually disagrees with them.""" + assert inspect.isfunction(InstrumentHealth.entitlement) + assert inspect.isfunction(InstrumentHealth.standings) + interval = _calibration(every_runs=100.0) + health = _health(interval, since=Accrued(runs=10.0)) + assert health.entitlement() is Entitlement.ENTITLED + assert health.entitlement(Campaign(runs=200)) is Entitlement.EXPIRING + + +# -- the refusal ---------------------------------------------------------------- + + +_FORBIDDEN = ("mtbf", "failure_rate", "remaining_life", "predict", "reliability", "hazard", "lifetime") + + +def test_no_public_callable_returns_a_failure_prediction(): + """Reliability prediction needs population data this package does not have. + + A name scan rather than a spot check, because the failure this guards against is + somebody adding a helpful `estimate_remaining_life` in six months. + """ + for name, obj in vars(durability).items(): + if name.startswith("_") or not callable(obj): + continue + if getattr(obj, "__module__", None) != durability.__name__: + continue + low = name.lower() + for token in _FORBIDDEN: + assert token not in low, f"durability exposes '{name}', which promises a prediction" + + +def test_no_dataclass_field_stores_a_failure_rate(): + """The same refusal applied to data, where an invented number would actually live.""" + for name, obj in vars(durability).items(): + if not (isinstance(obj, type) and dataclasses.is_dataclass(obj)): + continue + for field in dataclasses.fields(obj): + low = field.name.lower() + for token in _FORBIDDEN: + assert token not in low, f"{name}.{field.name} would hold a number nobody measured" + + +def test_the_module_names_what_it_refuses_and_what_each_would_take(): + """A refusal recorded as data is checkable; one left in prose is a claim.""" + assert refusals() is NOT_COMPUTED + assert len(NOT_COMPUTED) >= 3 + quantities = " ".join(r.quantity.lower() for r in NOT_COMPUTED) + assert "mean time between failures" in quantities + assert "failure rate" in quantities + assert "remaining useful life" in quantities + for r in NOT_COMPUTED: + assert r.why and r.what_it_would_take, f"'{r.quantity}' is refused without saying why" + + +# -- the workcell view ---------------------------------------------------------- + + +def test_this_repo_records_no_service_history_for_any_instrument(): + """Empty as a finding, not as an omission. + + A plausible service interval written into these tables would be quoted as this lab's + service interval within two releases, which is how an estimate becomes a specification. + """ + assert OBLIGATIONS == {} + assert SERVICE_HISTORY == {} + h = health_for("star") + assert h.obligations == () + assert h.entitlement() is Entitlement.UNKNOWN + + +def test_no_instrument_in_this_workcell_is_entitled_today(): + """The finding this module exists to produce, pinned so a regression is loud.""" + wc = Workcell.default() + rows = untrusted_instruments(wc) + expected = len(wc.present_keys()) + len(wc.federated) + assert len(rows) == expected + assert all(r.entitlement is Entitlement.UNKNOWN for r in rows) + assert all(r.restores for r in rows), "an untrusted instrument must name what would fix it" + + +def test_the_instrument_list_comes_from_the_workcell_not_the_health_records(): + """Iterating the health dict would report on the instruments somebody wrote down. + + Those are precisely the instruments this module is not about. + """ + wc = Workcell.default() + key = wc.present_keys()[0] + interval = _calibration(instrument=key, every_runs=100.0) + supplied = {key: _health(interval, since=Accrued(runs=1.0))} + rows = untrusted_instruments(wc, supplied) + names = {r.instrument for r in rows} + assert key not in names, "the one tracked instrument is entitled" + assert len(rows) == len(wc.present_keys()) + len(wc.federated) - 1 + for federated in wc.federated: + assert federated in names, "a federated instrument's service history is still this lab's problem" + + +def test_a_campaign_moves_instruments_out_of_entitled_without_changing_the_lab(): + """The same workcell, the same records, a longer plan, a different answer.""" + wc = Workcell.default() + key = wc.present_keys()[0] + interval = _calibration(instrument=key, every_runs=100.0) + supplied = {key: _health(interval, since=Accrued(runs=90.0))} + + short = entitlement_summary(wc, supplied, Campaign(runs=5)) + long = entitlement_summary(wc, supplied, Campaign(runs=50)) + assert short[Entitlement.ENTITLED.value] == 1 + assert long[Entitlement.ENTITLED.value] == 0 + assert long[Entitlement.EXPIRING.value] == 1 + assert short["total"] == long["total"] == len(wc.present_keys()) + len(wc.federated) + + +def test_every_instrument_lands_in_exactly_one_bucket(): + wc = Workcell.default() + counts = entitlement_summary(wc) + buckets = sum(counts[e.value] for e in Entitlement) + assert buckets == counts["total"] + assert counts[Entitlement.UNKNOWN.value] == counts["total"] diff --git a/tests/test_feedback.py b/tests/test_feedback.py new file mode 100644 index 0000000..6bf2474 --- /dev/null +++ b/tests/test_feedback.py @@ -0,0 +1,619 @@ +"""Device-free tests for the feedback layer. + +The tests that matter here try to get a loop reported as closed when it is not. Four ways +that happens, and they are kept apart on purpose because they cost four different things to +fix: the sensor sits downstream of the actuator so the correction never reaches the material, +nothing declares what good is so there is nothing to steer toward, the number comes from a +step that produces nothing, and the controller is weaker than the correction it would make. +Collapsing any two of those into one verdict would send a lab to spend one budget on two +problems. + +The other half of the file guards the refusal. This module must not start returning a gain, +a time constant, or a settling time, because it has no plant model and no measured response +data, and a number of that shape is exactly the kind that gets quoted into a specification. +""" + +from __future__ import annotations + +from enum import Enum + +import pytest + +from autonomous_lab import Workcell, build_ledger, protocols +from autonomous_lab.model import Protocol, Step, Verdict, ZeroDecodeOp +from autonomous_lab.qc import GATES, PROTOCOL_GATES, Basis +from autonomous_lab.workcell import InstrumentConfig +from autonomous_lab import feedback +from autonomous_lab.feedback import ( + LOOPS, + NOT_MODELLED, + Closable, + Closure, + Controller, + Correction, + Envelope, + Loop, + can_close, + envelope_from_criterion, + feedback_report, + in_flight_exposure, + ungated_protocols, +) + + +# -- fixtures ------------------------------------------------------------------ + + +def _env(quantity="q", lower=0.5, upper=None, basis=Basis.IN_HOUSE): + return Envelope(quantity=quantity, units="u", lower=lower, upper=upper, basis=basis) + + +def _loop( + measured_at, + corrects, + envelope=None, + correction=Correction.HOLD, + applied_by=Controller.CODED, + quantity="q", + name="test_loop", +): + return Loop( + name=name, + quantity=quantity, + measured_at=measured_at, + corrects=corrects, + correction=correction, + applied_by=applied_by, + envelope=envelope, + ) + + +def _two_automated_steps(): + """A protocol whose every step runs headless, so nothing but the loop can be at fault.""" + return Protocol( + name="tiny", + summary="test", + steps=( + Step(instrument="element_aviti", op=ZeroDecodeOp.PROBE_HTTP.value, summary="sensor"), + Step( + instrument="element_aviti", + op=ZeroDecodeOp.WATCH_RUN_FOLDER.value, + summary="actuator", + params={"run_dir": "/tmp/x"}, + ), + ), + ) + + +def _three_automated_steps(): + return Protocol( + name="tiny3", + summary="test", + steps=( + Step(instrument="element_aviti", op=ZeroDecodeOp.PROBE_HTTP.value, summary="sensor"), + Step(instrument="namocell", op=ZeroDecodeOp.DISCOVER_USB.value, summary="in between"), + Step( + instrument="element_aviti", + op=ZeroDecodeOp.WATCH_RUN_FOLDER.value, + summary="actuator", + params={"run_dir": "/tmp/x"}, + ), + ), + ) + + +def _blocked_sensor_protocol(): + """A sensor step the workcell cannot drive, sitting after the step it would correct.""" + return Protocol( + name="blocked", + summary="test", + steps=( + Step(instrument="namocell", op=ZeroDecodeOp.DISCOVER_USB.value, summary="actuator"), + Step(instrument="namocell", op="start_sort", summary="sensor"), + ), + ) + + +def _costed(protocol, wc=None): + wc = wc or Workcell.default() + return protocol, build_ledger(protocol, wc) + + +# -- the positive control ------------------------------------------------------ +# Without this the whole file could pass by refusing everything, which is the failure mode +# of every checker that has never been shown a case it should accept. + + +def test_a_well_formed_loop_closes(): + protocol, ledger = _costed(_two_automated_steps()) + loop = _loop("probe_http", "watch_run_folder", envelope=_env()) + closure = can_close(loop, protocol, ledger) + assert closure.verdict is Closable.CLOSES + assert closure.closes + assert closure.blockers == () + + +# -- refusal one: the sensor is downstream of the actuator --------------------- + + +def test_a_sensor_downstream_of_its_actuator_is_refused_with_that_reason(): + """The load-bearing check. A measurement taken after the correction point steers nothing. + + Everything else about this loop is right: it has an envelope, both steps run headless, and + a coded controller may make the correction. The only thing wrong is the order, and the + order alone must refuse it. + """ + protocol, ledger = _costed(_two_automated_steps()) + loop = _loop("watch_run_folder", "probe_http", envelope=_env()) + closure = can_close(loop, protocol, ledger) + assert closure.verdict is Closable.OPEN_LOOP_SENSOR_TOO_LATE + assert not closure.closes + assert closure.blockers == closure.blockers[:1], "no other leg of this loop is broken" + + +def test_the_structural_refusal_is_the_one_no_purchase_moves(): + assert Closable.OPEN_LOOP_SENSOR_TOO_LATE.structural + assert not Closable.NO_ENVELOPE.structural + assert not Closable.UNAVAILABLE_MEASUREMENT.structural + assert not Closable.UNPERMITTED_CORRECTION.structural + + +def test_measuring_at_the_step_being_corrected_is_open_too(): + """A step is atomic here, so a correction applied inside one is not visible to this layer. + + Reported as open rather than closed, because claiming a loop closes on a mechanism the + package does not model is the same overclaim as calling an unbenchmarked robot trusted. + """ + protocol, ledger = _costed(_two_automated_steps()) + loop = _loop("probe_http", "probe_http", envelope=_env()) + closure = can_close(loop, protocol, ledger) + assert closure.verdict is Closable.OPEN_LOOP_SENSOR_TOO_LATE + assert "atomic" in closure.reason + + +# -- refusal two: nothing to steer toward -------------------------------------- + + +def test_a_loop_steering_toward_nothing_is_refused_separately(): + """Distinct from lateness, and fixed by an experiment rather than by a reordering.""" + protocol, ledger = _costed(_two_automated_steps()) + loop = _loop("probe_http", "watch_run_folder", envelope=None) + closure = can_close(loop, protocol, ledger) + assert closure.verdict is Closable.NO_ENVELOPE + assert closure.verdict is not Closable.OPEN_LOOP_SENSOR_TOO_LATE + assert not closure.closes + + +def test_absence_of_an_envelope_never_reads_as_a_pass(): + """The skip-that-looks-like-a-pass case, in the shape this module can produce it. + + A loop declaring no target is not a loop with a permissive target. The natural + implementation checks the envelope only when one is present, and that implementation + reports every envelope-less loop as closed. + """ + protocol, ledger = _costed(_two_automated_steps()) + closure = can_close(_loop("probe_http", "watch_run_folder"), protocol, ledger) + assert not closure.closes + assert Closable.NO_ENVELOPE in closure.refusals() + + +def test_an_envelope_bounding_nothing_is_refused_at_construction(): + """An unbounded band contains every value, so a loop over it is in specification forever.""" + with pytest.raises(ValueError, match="bounds nothing"): + Envelope(quantity="q", units="u") + + +def test_an_inverted_envelope_is_refused(): + with pytest.raises(ValueError, match="below lower"): + Envelope(quantity="q", units="u", lower=2.0, upper=1.0) + + +def test_an_envelope_on_a_neighbouring_quantity_is_not_an_envelope_for_this_one(): + """A declaration that a summary would count and that bounds the wrong thing.""" + protocol, ledger = _costed(_two_automated_steps()) + loop = _loop("probe_http", "watch_run_folder", envelope=_env(quantity="something_else")) + closure = can_close(loop, protocol, ledger) + assert closure.verdict is Closable.NO_ENVELOPE + assert "something_else" in closure.reason + + +# -- refusal three: the number never arrives ----------------------------------- + + +def test_a_loop_reading_a_step_this_lab_cannot_run_is_refused_with_a_third_reason(): + protocol = protocols.get("single_cell_genomics") + ledger = build_ledger(protocol) + closure = can_close(LOOPS["quant_steers_flow_cell_loading"], protocol, ledger) + assert closure.verdict is Closable.UNAVAILABLE_MEASUREMENT + assert "read_absorbance" in closure.reason + + +def test_measurement_availability_is_resolved_from_the_ledger_not_asserted(): + """The same loop, two workcells, two answers -- so nothing here is hard-coded. + + A second implementation of cost_step living in this module would eventually disagree with + the ledger about whether a step produces a number, and a control layer that disagreed with + the ledger would be worse than no control layer. + """ + protocol = _two_automated_steps() + loop = _loop("probe_http", "watch_run_folder", envelope=_env()) + + full = build_ledger(protocol, Workcell.default()) + assert can_close(loop, protocol, full).closes + + bare = Workcell( + name="bare", + instruments={"element_aviti": InstrumentConfig(key="element_aviti", present=False)}, + ) + stripped = build_ledger(protocol, bare) + assert stripped.rows[0].verdict is Verdict.MANUAL + closure = can_close(loop, protocol, stripped) + assert closure.verdict is Closable.UNAVAILABLE_MEASUREMENT + + +def test_a_broken_instrument_and_an_unwired_one_refuse_alike_and_report_differently(): + """Wiring plr-tested in changes the reason and not the verdict. + + Both are UNAVAILABLE_MEASUREMENT because in both cases no number arrives. The reason is + carried through from the ledger because the next action differs completely: one is a + checkout path, the other is a deterministic timeout somebody has to debug. + """ + protocol = protocols.get("single_cell_genomics") + loop = LOOPS["quant_steers_flow_cell_loading"] + + unwired = build_ledger(protocol) + wired_wc = Workcell.default() + wired_wc.plr_tested_root = "/nonexistent/plr-tested" + wired = build_ledger(protocol, wired_wc) + + reader = next(i for i, s in enumerate(protocol.steps) if s.op == "read_absorbance") + assert unwired.rows[reader].verdict is Verdict.MANUAL + assert wired.rows[reader].verdict is Verdict.BROKEN + + assert can_close(loop, protocol, unwired).verdict is Closable.UNAVAILABLE_MEASUREMENT + wired_closure = can_close(loop, protocol, wired) + assert wired_closure.verdict is Closable.UNAVAILABLE_MEASUREMENT + assert "FAILED on the instrument" in wired_closure.reason + + +# -- refusal four: the controller may not make that correction ----------------- + + +def test_an_unpermitted_correction_is_a_fourth_and_separate_reason(): + protocol, ledger = _costed(_two_automated_steps()) + loop = _loop( + "probe_http", + "watch_run_folder", + envelope=_env(), + correction=Correction.DISCARD, + applied_by=Controller.MODEL, + ) + closure = can_close(loop, protocol, ledger) + assert closure.verdict is Closable.UNPERMITTED_CORRECTION + assert "human" in closure.reason + + +def test_stopping_is_the_only_correction_a_model_may_make_alone(): + """Stopping commits nothing, so the model may do it; everything else spends material.""" + assert Controller.MODEL.may_apply(Correction.HOLD) + for correction in (Correction.RETUNE, Correction.REPEAT, Correction.DIVERT, Correction.DISCARD): + assert not Controller.MODEL.may_apply(correction) + + +def test_only_stopping_is_reversible(): + assert Correction.HOLD.reversible + for correction in (Correction.RETUNE, Correction.REPEAT, Correction.DIVERT, Correction.DISCARD): + assert not correction.reversible + + +def test_authority_may_be_strengthened_and_never_weakened(): + assert Controller.HUMAN.may_apply(Correction.RETUNE) + assert Controller.CODED.may_apply(Correction.REPEAT) + assert not Controller.MODEL_PROPOSES.may_apply(Correction.DIVERT) + assert not Controller.CODED.may_apply(Correction.DISCARD) + + +def test_a_drafting_model_still_counts_as_a_model_in_the_path(): + assert Controller.MODEL.model_in_path + assert Controller.MODEL_PROPOSES.model_in_path + assert not Controller.CODED.model_in_path + assert not Controller.HUMAN.model_in_path + + +# -- the four are never collapsed ---------------------------------------------- + + +def test_the_four_refusals_stay_four_distinct_verdicts(): + """Three of these are the ones the module exists to keep apart, and each has its own fix.""" + protocol, ledger = _costed(_two_automated_steps()) + late = can_close(_loop("watch_run_folder", "probe_http", envelope=_env()), protocol, ledger) + targetless = can_close(_loop("probe_http", "watch_run_folder"), protocol, ledger) + unpermitted = can_close( + _loop( + "probe_http", + "watch_run_folder", + envelope=_env(), + correction=Correction.DISCARD, + applied_by=Controller.MODEL, + ), + protocol, + ledger, + ) + genomics = protocols.get("single_cell_genomics") + absent = can_close( + LOOPS["quant_steers_flow_cell_loading"], genomics, build_ledger(genomics) + ) + + verdicts = {late.verdict, targetless.verdict, unpermitted.verdict, absent.verdict} + assert len(verdicts) == 4 + assert Closable.CLOSES not in verdicts + + +def test_a_loop_broken_four_ways_reports_all_four_and_headlines_the_structural_one(): + """Reporting only the first refusal is how a lab fixes one and rediscovers the next.""" + protocol, ledger = _costed(_blocked_sensor_protocol()) + loop = _loop( + "start_sort", + "discover_usb", + envelope=None, + correction=Correction.DISCARD, + applied_by=Controller.MODEL, + ) + closure = can_close(loop, protocol, ledger) + assert closure.refusals() == ( + Closable.OPEN_LOOP_SENSOR_TOO_LATE, + Closable.NO_ENVELOPE, + Closable.UNAVAILABLE_MEASUREMENT, + Closable.UNPERMITTED_CORRECTION, + ) + assert closure.verdict is Closable.OPEN_LOOP_SENSOR_TOO_LATE + assert len(closure.all_reasons()) == 4 + + +# -- nothing here may declare itself closed ------------------------------------ + + +def test_no_field_on_a_loop_declares_it_closed(): + """Closability is computed against a protocol and a ledger, never set by an author. + + The same rule the ledger applies to automation and qc applies to readiness. If this ever + fails, somebody has added an override and every report in this module can be silenced. + """ + loop = LOOPS["pooled_quant_steers_pooling"] + for field in ("closes", "closed", "closable", "verdict", "reason", "blockers"): + assert field not in Loop.__dataclass_fields__ + assert not hasattr(loop, field) + + +def test_a_closure_is_derived_from_its_blockers_rather_than_stored_beside_them(): + """A stored verdict sitting next to the blockers it came from eventually disagrees.""" + assert "verdict" not in Closure.__dataclass_fields__ + assert "closes" not in Closure.__dataclass_fields__ + assert isinstance(getattr(Closure, "verdict"), property) + + +# -- the refusal to invent dynamics -------------------------------------------- + + +def test_nothing_public_offers_a_gain_a_time_constant_or_a_settling_time(): + """No plant model and no measured response data, so no dynamic quantity may be emitted. + + Checked by name across the module surface rather than argued in prose, because the way + this refusal breaks is somebody adding one convenient helper. + """ + forbidden = ( + "gain", + "overshoot", + "damping", + "settling", + "stability", + "time_constant", + "timeconstant", + "tau", + "rise_time", + "response_time", + ) + names = [] + for name, obj in vars(feedback).items(): + if name.startswith("_"): + continue + names.append(name) + if isinstance(obj, type) and getattr(obj, "__module__", "") == feedback.__name__: + names.extend(k for k in vars(obj) if not k.startswith("_")) + names.extend(getattr(obj, "__dataclass_fields__", {})) + if issubclass(obj, Enum): + names.extend(str(m.value) for m in obj) + offenders = [n for n in names if any(f in n.lower() for f in forbidden)] + assert offenders == [] + + +def test_the_quantities_this_module_will_not_emit_are_named(): + """An unstated omission is indistinguishable from an oversight.""" + for quantity in ("gain", "overshoot", "time constant", "settling time"): + assert quantity in NOT_MODELLED + + +def test_in_flight_exposure_is_a_plate_count_and_not_a_duration(): + """The honest form of 'how fast can this loop react', in the unit the protocol knows.""" + protocol = _three_automated_steps() + flight = in_flight_exposure(_loop("probe_http", "watch_run_folder", envelope=_env()), protocol) + assert flight.steps_between == 1 + assert flight.intervening == ("discover_usb",) + assert not flight.immediate + assert "plate" in flight.meaning + assert "seconds" in flight.meaning and "ever been timed" in flight.meaning + assert "steps_between" in type(flight).__dataclass_fields__ + assert not any( + f.endswith("_s") or "seconds" in f for f in type(flight).__dataclass_fields__ + ) + + +def test_a_correction_landing_at_the_next_step_commits_nothing_in_between(): + protocol = _two_automated_steps() + flight = in_flight_exposure(_loop("probe_http", "watch_run_folder", envelope=_env()), protocol) + assert flight.steps_between == 0 + assert flight.immediate + + +def test_in_flight_exposure_refuses_a_loop_that_never_reaches_its_material(): + """Zero would read as the best possible case rather than as the absence of a loop.""" + protocol = _two_automated_steps() + with pytest.raises(ValueError, match="measures at or after"): + in_flight_exposure(_loop("watch_run_folder", "probe_http", envelope=_env()), protocol) + + +def test_in_flight_exposure_on_the_real_protocol_counts_protocol_structure(): + protocol = protocols.get("single_cell_genomics") + flight = in_flight_exposure(LOOPS["occupancy_steers_amplification"], protocol) + assert flight.steps_between == 1 + assert flight.intervening == ("wait_complete",) + + +# -- refusing to resolve what it cannot resolve -------------------------------- + + +def test_an_op_used_twice_in_a_flow_is_not_a_position(): + """Picking the first occurrence reports a loop as closed whenever the wrong one is upstream.""" + protocol = protocols.get("single_cell_genomics") + ledger = build_ledger(protocol) + loop = _loop("manual_load", "start_run", envelope=_env()) + with pytest.raises(ValueError, match="appears 2 times"): + can_close(loop, protocol, ledger) + + +def test_a_loop_naming_a_step_the_protocol_does_not_have_is_refused(): + protocol, ledger = _costed(_two_automated_steps()) + with pytest.raises(KeyError, match="not a step"): + can_close(_loop("no_such_step", "watch_run_folder", envelope=_env()), protocol, ledger) + + +def test_a_ledger_built_for_another_protocol_is_refused(): + """A verdict mixed from two protocols describes neither, and would silently use the + wrong step's verdict for the measurement.""" + protocol = _two_automated_steps() + other = build_ledger(protocols.get("single_cell_genomics")) + with pytest.raises(ValueError, match="built for protocol"): + can_close(_loop("probe_http", "watch_run_folder", envelope=_env()), protocol, other) + + +# -- envelopes come from qc, never from here ----------------------------------- + + +def test_an_envelope_is_resolved_from_the_qc_criterion_rather_than_restated(): + """Two copies of one cutoff eventually disagree and nothing reveals which is live.""" + gate = GATES["sort_occupancy_before_amplification"] + criterion = gate.criteria[0] + envelope = envelope_from_criterion(gate, criterion.name) + assert envelope.lower == criterion.bound + assert envelope.quantity == criterion.measurement + assert envelope.basis is criterion.basis + assert envelope.units == "fraction" + + declared = LOOPS["occupancy_steers_amplification"].envelope + assert declared is not None + assert declared.lower == criterion.bound + + +def test_a_two_sided_criterion_becomes_a_two_sided_envelope(): + gate = GATES["library_quant_before_flow_cell"] + envelope = envelope_from_criterion(gate, "not_protein_contaminated") + assert envelope.lower == 1.7 + assert envelope.upper == 2.1 + assert envelope.contains(1.85) + assert not envelope.contains(2.5) + + +def test_an_envelope_over_a_criterion_qc_does_not_declare_is_refused(): + with pytest.raises(KeyError, match="declares no criterion"): + envelope_from_criterion(GATES["library_quant_before_flow_cell"], "invented") + + +def test_an_unvalidated_envelope_is_reported_and_does_not_refuse_the_loop(): + """A loop around a vendor cutoff genuinely closes -- it just closes on somebody else's band. + + Making this a fifth refusal would say the fix is the same as for a loop with no target at + all. It is not: this one needs a titration, and the loop works meanwhile. + """ + protocol, ledger = _costed(_two_automated_steps()) + loop = _loop("probe_http", "watch_run_folder", envelope=_env(basis=Basis.VENDOR)) + closure = can_close(loop, protocol, ledger) + assert closure.closes + assert not loop.envelope.validated + + +# -- the real protocols -------------------------------------------------------- + + +def test_the_library_quant_cannot_steer_the_pool_it_is_taken_after(): + """The flagship finding, pinned so a regression is loud. + + Normalizing a pool is exactly what a concentration measurement is for, and this protocol + measures after the pool. Repairing the plate reader does not close this; measuring per + well before pooling does, and that is a protocol change rather than a purchase. + """ + protocol = protocols.get("single_cell_genomics") + ledger = build_ledger(protocol) + closure = can_close(LOOPS["pooled_quant_steers_pooling"], protocol, ledger) + assert closure.verdict is Closable.OPEN_LOOP_SENSOR_TOO_LATE + assert closure.measured_index > closure.corrects_index + assert "next-run adjustment" in closure.reason + + +def test_yield_read_off_a_finished_run_is_not_a_control_loop(): + """What a lab means by 'the system learns from its runs'. Real, useful, and not control.""" + protocol = protocols.get("single_cell_genomics") + ledger = build_ledger(protocol) + closure = can_close(LOOPS["yield_steers_resequencing"], protocol, ledger) + assert closure.verdict is Closable.OPEN_LOOP_SENSOR_TOO_LATE + + +def test_no_declared_loop_closes_in_this_lab_today(): + """The finding this module exists to produce. Every proposed loop fails, for three reasons.""" + protocol = protocols.get("single_cell_genomics") + report = feedback_report(protocol, build_ledger(protocol)) + assert len(report.rows) == 5 + assert report.closable() == [] + assert not report.closes_any() + assert len(report.structurally_open()) == 2 + + +def test_the_chemistry_loop_has_no_target_because_qc_declares_no_gate(): + """Resolved from the qc layer rather than asserted here, so it flips when a gate lands.""" + assert PROTOCOL_GATES["small_molecule_qc"] == () + assert "small_molecule_qc" in ungated_protocols() + protocol = protocols.get("small_molecule_qc") + report = feedback_report(protocol, build_ledger(protocol)) + assert [r.verdict for r in report.rows] == [Closable.NO_ENVELOPE] + + +def test_a_correction_nobody_can_apply_is_reported_beside_the_verdict_not_inside_it(): + """Whether the corrected step runs is the ledger's question and stays there. + + Folding it into the closure verdict would put an execution problem and a control problem + behind one word, and the lab would spend one budget on two things. + """ + protocol = protocols.get("single_cell_genomics") + report = feedback_report(protocol, build_ledger(protocol)) + stranded = report.corrections_nobody_can_apply() + assert stranded, "every correction in this protocol lands on a step no code path reaches" + for closure in stranded: + assert closure.corrects_verdict not in (Verdict.AUTOMATED, Verdict.SUPERVISED) + # The reported verdict is about the loop. None of these rows was refused for the + # reachability of the step it corrects, which is the separation being pinned. + assert closure.verdict is not Closable.UNPERMITTED_CORRECTION or not closure.loop.permitted() + + +def test_every_declared_envelope_here_rests_on_something_this_lab_has_not_validated(): + """Uncomfortable and correct, and the same answer qc gives about the same thresholds.""" + protocol = protocols.get("single_cell_genomics") + report = feedback_report(protocol, build_ledger(protocol)) + assert len(report.unvalidated_envelopes()) == len(report.rows) + + +def test_report_counts_account_for_every_row(): + protocol = protocols.get("single_cell_genomics") + report = feedback_report(protocol, build_ledger(protocol)) + counts = report.counts() + assert counts["total"] == len(report.rows) + assert sum(counts[reason.value] for reason in Closable) == counts["total"] + assert set(report.by_reason()) <= {reason.value for reason in Closable} diff --git a/tests/test_teaching.py b/tests/test_teaching.py new file mode 100644 index 0000000..9a290a4 --- /dev/null +++ b/tests/test_teaching.py @@ -0,0 +1,468 @@ +"""Device-free tests for the teaching layer: expert demonstrations and machine parity. + +Four failures are being hunted here, and they are the four ways a transfer report starts +flattering the lab it describes. + +A tolerance appearing out of one demonstration. One run establishes a value and no spread, +and every convention that would widen it -- a percentage, a sigma, a rule of thumb -- puts a +number in the report that no measurement supports. The refusal has to survive n=1 and n=2, +and it has to be reported rather than swallowed. + +A single good machine run reading as parity. This is the demo-to-assay failure in its purest +form: the number lands inside the expert's range, the report says MEETS, and nothing +anywhere records that the evidence could not distinguish that run from a lucky one. + +A vacuous pass. A transfer report handed no operations satisfies every `all()` in it, and an +empty demonstration queue over no protocols looks exactly like a lab that has finished +teaching its machines. Both must read as nothing-was-checked. + +And a ranking that is really a list. The queue of what an SME should demonstrate next has to +fall out of the protocols it is given, so that handing it different protocols changes the +order. A hardcoded order is advice about somebody else's lab. +""" + +from __future__ import annotations + +import dataclasses + +import pytest + +from autonomous_lab import protocols +from autonomous_lab.intelligence import BENCHMARKS_BY_OP +from autonomous_lab.model import Protocol, Step +from autonomous_lab.teaching import ( + DEMONSTRATIONS, + MACHINE_OBSERVATIONS, + MIN_DEMONSTRATIONS, + TRANSFERABLE, + TRANSFERABLE_BY_OP, + Attainment, + Demonstration, + Envelope, + Goal, + MachineObservation, + attainment, + demonstration_queue, + demonstrations_still_needed, + taught, + teaching_summary, + transfer_report, + untaught_operations, +) + +CLEANUP = "pcr_enrichment_round1_cleanup" # HIGHER: recovered fraction +POOLING = "library_pool" # LOWER: representation CV +LYSIS = "wgs_prep_lysis" # WINDOW: delivered volume + + +def _demo(value: float, op: str = CLEANUP, by: str = "scientist_a", conditions=None) -> Demonstration: + spec = TRANSFERABLE_BY_OP[op] + return Demonstration( + operation=op, + metric=spec.metric, + units=spec.units, + value=value, + by=by, + conditions=conditions if conditions is not None else spec.conditions, + ) + + +def _obs(value: float, op: str = CLEANUP, conditions=None) -> MachineObservation: + spec = TRANSFERABLE_BY_OP[op] + return MachineObservation( + operation=op, + metric=spec.metric, + units=spec.units, + value=value, + by="a run card", + conditions=conditions if conditions is not None else spec.conditions, + ) + + +def _envelope(values, op: str = CLEANUP, by="scientist_a") -> Envelope: + spec = TRANSFERABLE_BY_OP[op] + handles = [by] * len(values) if isinstance(by, str) else list(by) + return Envelope( + operation=op, + metric=spec.metric, + units=spec.units, + goal=spec.goal, + demonstrations=tuple(_demo(v, op=op, by=h) for v, h in zip(values, handles)), + ) + + +# -- one demonstration is a value, not a tolerance ----------------------------- + + +def test_one_demonstration_gives_a_value_and_no_tolerance(): + """The whole discipline of the module in one assertion.""" + env = _envelope([0.82]) + assert env.center() == 0.82 + assert env.tolerance() is None + assert env.spread() is None + + +def test_two_demonstrations_still_give_no_tolerance(): + """Two points have one degree of freedom; the interval they imply IS the two points.""" + env = _envelope([0.82, 0.88]) + assert env.center() == pytest.approx(0.85) + assert env.tolerance() is None + + +def test_the_refusal_is_reported_and_says_how_many_more_are_needed(): + """A refusal nobody can read is indistinguishable from an answer nobody checked.""" + env = _envelope([0.82]) + refusal = env.refusal() + assert refusal is not None + assert str(MIN_DEMONSTRATIONS) in refusal + assert str(MIN_DEMONSTRATIONS - 1) in refusal + + +def test_an_envelope_over_no_demonstrations_states_nothing_at_all(): + """Handed nothing, it must not compute anything -- the vacuous-pass failure at the source.""" + env = _envelope([]) + assert env.n() == 0 + assert env.center() is None + assert env.tolerance() is None + assert env.caveat() is None + assert "no recorded demonstration" in (env.refusal() or "") + + +def test_the_tolerance_is_the_observed_range_and_nothing_wider(): + """min and max of what happened. A multiplier here would be a number with no author.""" + env = _envelope([0.80, 0.90, 0.85]) + assert env.tolerance() == (0.80, 0.90) + assert env.spread() == pytest.approx(0.10) + + +def test_an_envelope_from_one_demonstrator_carries_that_caveat(): + """Three runs by one person describe the person, and the claim has to travel with it.""" + env = _envelope([0.80, 0.85, 0.90], by="scientist_a") + assert env.tolerance() is not None + assert "one" in (env.caveat() or "") + wide = _envelope([0.80, 0.85, 0.90], by=("scientist_a", "scientist_b", "scientist_c")) + assert wide.caveat() is None + + +# -- one good machine run is not parity ---------------------------------------- + + +def test_one_good_machine_run_does_not_report_meets(): + """The demo-to-assay failure. The value lands inside the range and proves nothing.""" + env = _envelope([0.80, 0.85, 0.90]) + parity = attainment(env, [_obs(0.88)]) + assert parity.attainment is Attainment.INDISTINGUISHABLE_FROM_UNMEASURED + assert not parity.demonstrated + assert "lucky" in parity.reason + + +def test_two_good_machine_runs_are_still_not_meets(): + env = _envelope([0.80, 0.85, 0.90]) + parity = attainment(env, [_obs(0.88), _obs(0.87)]) + assert parity.attainment is Attainment.INDISTINGUISHABLE_FROM_UNMEASURED + assert parity.machine_n == 2 + + +def test_enough_machine_runs_inside_the_range_do_report_meets(): + """The module has to be able to say yes, or its refusals mean nothing.""" + env = _envelope([0.80, 0.85, 0.90]) + parity = attainment(env, [_obs(0.88), _obs(0.87), _obs(0.91)]) + assert parity.attainment is Attainment.MEETS + assert parity.demonstrated + assert parity.expert_n == 3 and parity.machine_n == 3 + + +def test_a_machine_nobody_measured_is_unmeasured_rather_than_below(): + """A skip must not read as a result in either direction.""" + env = _envelope([0.80, 0.85, 0.90]) + parity = attainment(env, []) + assert parity.attainment is Attainment.UNMEASURED + assert not parity.demonstrated + assert parity.machine_n == 0 + + +def test_parity_is_judged_on_the_worst_run_not_the_mean(): + """A mean lets one excellent run pay for a bad one; the bad plate is the one that costs.""" + env = _envelope([0.80, 0.85, 0.90]) + runs = [_obs(0.98), _obs(0.98), _obs(0.55)] + assert sum(o.value for o in runs) / 3 >= 0.80 # the mean would have passed + assert attainment(env, runs).attainment is Attainment.BELOW + + +def test_a_machine_measured_under_other_conditions_is_unmeasured(): + """A picogram run against a nanogram demonstration is not a comparison.""" + env = _envelope([0.80, 0.85, 0.90]) + elsewhere = [_obs(0.99, conditions="ten times the input amount") for _ in range(5)] + parity = attainment(env, elsewhere) + assert parity.attainment is Attainment.UNMEASURED + assert "conditions" in parity.reason + + +def test_a_thin_envelope_cannot_be_attained_by_any_number_of_machine_runs(): + """The mirror case: the machine side is fine and there is nothing to place it against.""" + env = _envelope([0.85]) + parity = attainment(env, [_obs(0.88) for _ in range(20)]) + assert parity.attainment is Attainment.INDISTINGUISHABLE_FROM_UNMEASURED + assert parity.expert_n == 1 and parity.machine_n == 20 + + +def test_no_envelope_at_all_is_unmeasured_not_meets(): + parity = attainment(None, [_obs(0.99) for _ in range(10)]) + assert parity.attainment is Attainment.UNMEASURED + assert parity.expert_n == 0 + + +def test_only_meets_reads_as_parity(): + """Three ways of not knowing are not degrees of knowing.""" + assert Attainment.MEETS.demonstrated_parity + for other in ( + Attainment.BELOW, + Attainment.UNMEASURED, + Attainment.INDISTINGUISHABLE_FROM_UNMEASURED, + ): + assert not other.demonstrated_parity + + +# -- direction: better is not always bigger ------------------------------------ + + +def test_a_machine_beating_the_expert_on_a_lower_is_better_metric_meets(): + env = _envelope([10.0, 12.0, 14.0], op=POOLING) + assert TRANSFERABLE_BY_OP[POOLING].goal is Goal.LOWER + parity = attainment(env, [_obs(v, op=POOLING) for v in (5.0, 6.0, 7.0)]) + assert parity.attainment is Attainment.MEETS + + +def test_overshooting_a_two_sided_metric_is_not_better_performance(): + """Delivering 11 uL where 10 was demonstrated is a different failure, not an improvement.""" + env = _envelope([9.8, 10.0, 10.2], op=LYSIS) + assert TRANSFERABLE_BY_OP[LYSIS].goal.outside_is_always_worse + parity = attainment(env, [_obs(v, op=LYSIS) for v in (11.0, 11.1, 11.2)]) + assert parity.attainment is Attainment.BELOW + # And the same numbers would have passed a one-sided comparison, which is the bug. + assert min(11.0, 11.1, 11.2) >= env.tolerance()[0] + + +# -- taught, and the backlog --------------------------------------------------- + + +def test_an_operation_nobody_demonstrated_is_untaught_rather_than_passing(): + ok, why = taught(CLEANUP) + assert not ok + assert "zero recorded" in why + + +def test_an_operation_with_no_specification_at_all_is_untaught_and_says_so(): + ok, why = taught("an_operation_nobody_ever_wrote_down") + assert not ok + assert "as good as the expert" in why + + +def test_every_operation_in_this_repo_is_untaught(): + """A coverage claim the module's own data would contradict. There are no demonstrations.""" + assert DEMONSTRATIONS == () + assert MACHINE_OBSERVATIONS == () + for op in TRANSFERABLE_BY_OP: + ok, _why = taught(op) + assert not ok, f"'{op}' reports taught with no demonstration recorded" + assert teaching_summary()["operations_with_envelope"] == 0 + + +def test_untaught_operations_covers_every_distinct_step_of_the_genomics_protocol(): + protocol = protocols.get("single_cell_genomics") + untaught = untaught_operations(protocol) + assert len(untaught) == len({s.op for s in protocol.steps}) + + +def test_every_specified_operation_is_one_a_reference_protocol_actually_runs(): + """A backlog naming work no flow needs is a list somebody will do and nobody will use.""" + ops = set() + for name in ("single_cell_genomics", "small_molecule_qc"): + ops.update(s.op for s in protocols.get(name).steps) + for t in TRANSFERABLE: + assert t.op in ops, f"'{t.op}' is specified for demonstration and no protocol runs it" + + +def test_demonstrations_still_needed_is_the_full_minimum_when_there_are_none(): + assert demonstrations_still_needed(CLEANUP) == MIN_DEMONSTRATIONS + assert demonstrations_still_needed("an_operation_nobody_ever_wrote_down") == MIN_DEMONSTRATIONS + + +# -- the queue is computed, not listed ----------------------------------------- + + +def test_the_queue_ranks_the_station_whose_sitting_settles_the_most(): + queue = demonstration_queue([protocols.get("single_cell_genomics")]) + by_instrument = {q.instrument: len(q.operations) for q in queue} + assert by_instrument["star"] == max(by_instrument.values()) + assert queue[0].instrument == "star" + assert queue[0].cost == sum(demonstrations_still_needed(op) for op in queue[0].operations) + + +def test_the_ranking_follows_the_protocols_it_is_given(): + """Move the same operations to another station and the queue reorders. + + This is what separates a computed ranking from a table someone wrote in a plausible order. + """ + synthetic = Protocol( + name="synthetic", + summary="the same operations at a different station", + steps=( + Step(instrument="namocell", op="start_sort", summary=""), + Step(instrument="namocell", op=POOLING, summary=""), + Step(instrument="star", op=LYSIS, summary=""), + ), + ) + queue = demonstration_queue([synthetic]) + assert queue[0].instrument == "namocell" + assert set(queue[0].operations) == {"start_sort", POOLING} + + +def test_a_different_protocol_set_gives_a_different_queue(): + chemistry = demonstration_queue([protocols.get("small_molecule_qc")]) + assert [q.instrument for q in chemistry] == ["viaflo96"] + both = demonstration_queue( + [protocols.get("single_cell_genomics"), protocols.get("small_molecule_qc")] + ) + assert "viaflo96" in {q.instrument for q in both} + assert both[0].instrument == "star" + + +def test_the_top_of_the_queue_is_not_the_first_thing_declared(): + """A ranking that agreed with declaration order would be untestable as a ranking.""" + first_declared = TRANSFERABLE[0].op + queue = demonstration_queue([protocols.get("single_cell_genomics")]) + assert first_declared not in queue[0].operations + + +def test_an_empty_queue_over_no_protocols_is_not_a_finished_programme(): + """Nothing to rank must not read as nothing left to do.""" + assert demonstration_queue([]) == [] + assert not transfer_report([]).transfers() + assert demonstration_queue([protocols.get("single_cell_genomics")]) + + +# -- the transfer report ------------------------------------------------------- + + +def test_a_report_over_no_operations_does_not_report_a_transfer(): + """`all()` over an empty list is True, which is how a report certifies a lab it never read.""" + report = transfer_report([]) + assert report.rows == [] + assert not report.transfers() + assert report.counts()["operations"] == 0 + + +def test_the_genomics_report_attains_nothing_and_names_every_operation(): + protocol = protocols.get("single_cell_genomics") + report = transfer_report([s.op for s in protocol.steps]) + assert report.counts()["operations"] == len({s.op for s in protocol.steps}) + assert report.attained() == [] + assert not report.transfers() + assert len(report.untaught()) == len(report.rows) + + +def test_benchmarks_are_resolved_from_intelligence_rather_than_restated(): + """Two statements of one target eventually disagree, and nothing reveals which is wrong.""" + report = transfer_report([CLEANUP]) + row = report.rows[0] + assert row.benchmarks + assert list(row.benchmarks) == list(BENCHMARKS_BY_OP[CLEANUP]) + assert row.benchmarks[0] is BENCHMARKS_BY_OP[CLEANUP][0] + + +def test_asserted_only_names_targets_nobody_has_shown_are_reachable(): + """A benchmark with no envelope: a number with a unit that no person has hit here.""" + protocol = protocols.get("single_cell_genomics") + report = transfer_report([s.op for s in protocol.steps]) + asserted = {r.operation for r in report.asserted_only()} + assert CLEANUP in asserted + assert "start_sort" in asserted + for row in report.asserted_only(): + assert not row.has_expert_envelope + + +def test_the_counts_do_not_claim_more_than_the_rows_hold(): + protocol = protocols.get("single_cell_genomics") + report = transfer_report([s.op for s in protocol.steps]) + counts = report.counts() + assert counts["with_envelope"] == 0 + assert counts["attained"] == 0 + assert counts["specified"] <= counts["operations"] + assert counts["asserted_only"] <= counts["operations"] + + +def test_teaching_summary_matches_the_module_data(): + s = teaching_summary() + assert s["operations_specified"] == len(TRANSFERABLE) + assert s["demonstrations_recorded"] == len(DEMONSTRATIONS) + assert s["machine_observations_recorded"] == len(MACHINE_OBSERVATIONS) + assert s["minimum_demonstrations"] == MIN_DEMONSTRATIONS + + +# -- nothing an author writes can silence the refusal -------------------------- + + +def test_no_field_anywhere_declares_a_tolerance_or_an_attainment(): + """The author-settable field that quiets a report is the failure this rules out. + + Every number an envelope states is a method over the demonstrations, so there is no + attribute to set and no stored value to drift from the data it summarizes. + """ + fields = {f.name for f in dataclasses.fields(Envelope)} + assert fields == {"operation", "metric", "units", "goal", "demonstrations"} + for name in ("tolerance", "center", "spread", "refusal", "caveat", "n"): + assert callable(getattr(Envelope, name)), f"{name} must be derived, not stored" + spec_fields = {f.name for f in dataclasses.fields(TRANSFERABLE_BY_OP[CLEANUP])} + for forbidden in ("taught", "tolerance", "attainment", "demonstrated"): + assert forbidden not in spec_fields + + +def test_an_envelope_refuses_to_pool_two_different_metrics(): + """One range over two quantities is a number about nothing.""" + spec = TRANSFERABLE_BY_OP[CLEANUP] + wrong_units = Demonstration( + operation=CLEANUP, + metric=spec.metric, + units="percent", + value=85.0, + by="scientist_b", + conditions=spec.conditions, + ) + with pytest.raises(ValueError, match="two metrics pooled"): + Envelope( + operation=CLEANUP, + metric=spec.metric, + units=spec.units, + goal=spec.goal, + demonstrations=(_demo(0.85), _demo(0.80), wrong_units), + ) + + +def test_an_envelope_refuses_a_demonstration_of_another_operation(): + spec = TRANSFERABLE_BY_OP[CLEANUP] + with pytest.raises(ValueError, match="holds a demonstration of"): + Envelope( + operation=CLEANUP, + metric=spec.metric, + units=spec.units, + goal=spec.goal, + demonstrations=(_demo(0.85), _demo(9.9, op=LYSIS)), + ) + + +def test_an_untaught_operation_is_named_in_its_own_refusal(): + """The untaught case is the common one, and it is the case where neither the envelope nor + the observations carry the operation's name -- both are absent. Without the caller + supplying it the refusal reads "no expert has demonstrated 'unknown'", which names nothing + and is useless as a queue entry, in the report whose entire job is to be that queue. + """ + report = transfer_report(["library_pool"]) + row = report.rows[0] + assert row.parity.operation == "library_pool" + assert "library_pool" in row.parity.reason + assert "unknown" not in row.parity.reason + + # And the bare function still works when the caller does not supply it. + assert attainment(None, (), operation="named_op").operation == "named_op" + assert attainment(None, ()).operation == "unknown" From 3330cdaae0dc8b80b08fe637addf9a3a31d4d846 Mon Sep 17 00:00:00 2001 From: di-omics Date: Tue, 28 Jul 2026 03:54:32 -0700 Subject: [PATCH 2/2] Close the vacuous passes the adversarial audit found Three audit lenses ran against each of the four modules -- one hunting defaults that read as a PASS when they are really a SKIP, one checking that no fact is recomputed where a sibling module already owns it, and one actively trying to extract a flattering answer. Between them they found the same class of defect this package exists to refuse, committed by the modules written to refuse it. The pattern repeats across all four. A gate reading a measurement nobody declared counted as coverage. A demand was met by a measurement set that says nothing. A duplicate gate row silenced an unsatisfiable one because the lookup returned the first match, which also made the report depend on row order. A workcell declaring no instrument at all reported as a clean lab. A NaN accrual was read as a measurement. A negative accrual minted budget that was never earned. A campaign landing exactly on a service boundary was reported as not crossing it. A coverage report holding nothing destructive reported as complete. Two are worth naming separately because they are cross-module rather than local. A recovery report built against a different gate report was accepted and silently mixed two workcells' worth of evidence; it is now refused. And a gate path this module had already refused was being republished as a residual detection path, so a failure this layer called uncovered appeared one line later as watched by the thing that could not watch it. Every fix carries a regression test that fails before it and passes after. No existing test was weakened to make anything pass. 63 new tests, 347 total. ruff clean. --- README.md | 2 +- autonomous_lab/coverage.py | 405 ++++++++++++++++--- autonomous_lab/durability.py | 419 ++++++++++++++++++-- autonomous_lab/feedback.py | 609 +++++++++++++++++++++++++---- autonomous_lab/teaching.py | 733 +++++++++++++++++++++++++++++++---- tests/test_coverage.py | 443 ++++++++++++++++++++- tests/test_durability.py | 348 +++++++++++++++++ tests/test_feedback.py | 148 +++++-- tests/test_teaching.py | 516 +++++++++++++++++++++++- 9 files changed, 3318 insertions(+), 305 deletions(-) diff --git a/README.md b/README.md index cc0b7ef..3dd20e7 100644 --- a/README.md +++ b/README.md @@ -554,7 +554,7 @@ event receiver and silently steals the first one's callbacks. pip install -e '.[dev]' && pytest ``` -283 device-free tests. The ones that matter most try to make the layer lie: +351 device-free tests. The ones that matter most try to make the layer lie: - claim a step is automated when its command is undecoded; claim a decoded command is runnable while its siblings are not; claim a federated leg runs when no run card was ever diff --git a/autonomous_lab/coverage.py b/autonomous_lab/coverage.py index 0cd157f..5ad2b72 100644 --- a/autonomous_lab/coverage.py +++ b/autonomous_lab/coverage.py @@ -31,21 +31,28 @@ destroyed silently -- and reports it in the exact language a reviewer would accept. COVERED here means covered by something that works with nobody in the room, and only two -things qualify: a deployed and validated visual check, and an evaluable gate reading an +things qualify: a deployed and validated visual check, and a gate that fires headless on an appropriate assay. An operator noticing is not coverage in a package about unattended -operation; it is the condition this whole package exists to remove. Data analysis after -sequencing is not coverage of a sample either, because the flow cell is already spent by the -time it speaks. Both are real all the same, so every row carries the path `recovery` -resolved, and nothing here ever claims that nothing anywhere would notice when a sibling -layer says something would. The claim is narrower and worth more: nothing MACHINE-CHECKABLE -stands between this failure and the material. +operation; it is the condition this whole package exists to remove, and a SUPERVISED gate is +that same operator holding a pipette in one hand and a screen in the other. Data analysis +after sequencing is not coverage of a sample either, because the flow cell is already spent +by the time it speaks. All of those are real all the same, so every row carries the path +`recovery` resolved, and nothing here ever claims that nothing anywhere would notice when a +sibling layer says something would. The claim is narrower and worth more: NO DECLARED +DETECTOR AND NO EVALUABLE GATE stands between this failure and the material. It is +deliberately not the broader claim that nothing machine-checkable does -- an instrument +raising USBError before it moves is machine-checkable and is not a detector this module +counts -- and `residual` is where an instrument-raised or post-hoc path is reported instead. `sota_lift` is the buying decision. Given a better vision capability it reports which failures move from uncovered to covered -- and, more prominently, which do not move at ANY capability, because they are INVISIBLE. The second list is the point of the function. Upgrading a model has a ceiling, the ceiling is physics rather than engineering, and a lab reading only the first list will keep buying model upgrades against failures no model -resolves. +resolves. A third list is required for the same reason the second one is: a real proposal is +partial, and a failure that a camera could reach and that THIS proposal does not reach is +neither lifted nor at the ceiling. Reported in neither list it disappears, and the report +becomes the purchase justification the second list was added to prevent. `mandatory_gates` states the same fact from the other side, as a demand rather than a gap. For an INVISIBLE failure that destroys material a gate is not one option among several; it @@ -60,7 +67,7 @@ from typing import Dict, List, Optional, Tuple from .ledger import build_ledger -from .qc import MEASUREMENTS, GateReport, Measurement, gate_report +from .qc import MEASUREMENTS, GateReport, Measurement, Readiness, gate_report from .recovery import ( FAILURE_MODES, Detection, @@ -90,6 +97,48 @@ def _destroys_material(failure: FailureMode) -> bool: return probe.destructive_and_silent +class GateStanding(str, Enum): + """What a declared gate actually does for one failure in one workcell. + + A boolean would collapse six situations that call for six different actions, and two of + the collapses are how a coverage report starts lying. The first is SUPERVISED read as + evaluable: `qc.Readiness.evaluable` is true for it, because from qc's side the number does + arrive, and it arrives with a human at the instrument -- which is the exact condition an + unattended workcell is built to remove. The second is UNMEASURED read as appropriate: a + gate whose measurement nobody declared has an unknown assay, and unknown is not a pass + here any more than it is in `qc.evaluate`. + + DEAD and ABSENT are separated for the same reason `Uncovered` separates its two members. + A dead gate is an instrument repair. A gate that is not on the protocol is an assay + somebody still has to design, and the two are not the same budget. + """ + + COUNTS = "counts" # evaluable with nobody in the room, on an assay that suits the sample + SUPERVISED = "supervised" # a number arrives, and only with a human at the instrument + UNMEASURED = "unmeasured" # the assay it reads was never declared, so it is unverified + WRONG_ASSAY = "wrong_assay" # a real, precise number about something other than the sample + DEAD = "dead" # unsatisfiable: no number arrives at all + ABSENT = "absent" # no gate on this protocol reads what this failure changes + + @property + def counts_as_coverage(self) -> bool: + """The load-bearing line: the one member that holds a failure with nobody in the room.""" + return self is GateStanding.COUNTS + + @property + def reveals_the_failure(self) -> bool: + """Whether the gate would surface THIS failure at all, coverage or not. + + SUPERVISED is the only member that is not coverage and still reveals: a person reads + the number and the failure is found. That is why it is reported as a residual path + rather than dropped -- the module refuses to count it and equally refuses to claim + nobody would notice. WRONG_ASSAY is on the other side of the line and it is worth being + precise about why: the number arrives, the gate passes, and it passes on an empty + library, so the path does not merely fail to be unattended, it fails to detect. + """ + return self in (GateStanding.COUNTS, GateStanding.SUPERVISED) + + class Cover(str, Enum): """How a failure is actually caught in a given workcell. @@ -105,13 +154,19 @@ class Cover(str, Enum): """ VISION = "vision" # a declared visual check is deployed and validated here - GATE = "gate" # a declared gate is evaluable and reads an appropriate assay + GATE = "gate" # a declared gate fires headless and reads an appropriate assay BOTH = "both" # either alone would hold it NEITHER = "neither" # no camera and no gate; the row says which kind of nothing @property def covered(self) -> bool: - """True where something machine-checkable stands between the failure and the material.""" + """True where a deployed visual check or an evaluable gate stands before the material. + + Narrower than "something machine-checkable", and the narrowness is deliberate rather + than an oversight. An instrument that raises before it moves is machine-checkable and + is not one of these two; it is reported on the row as a residual path. A NEITHER row + means no declared detector and no gate this module counts, never that the lab is blind. + """ return self is not Cover.NEITHER @property @@ -125,9 +180,14 @@ def closed_by(self) -> str: if self is Cover.VISION: return "a deployed, validated visual check" if self is Cover.GATE: - return "an evaluable gate reading an assay appropriate to the sample" + return ( + "a gate that fires with nobody at the instrument, reading an assay appropriate to " + "the sample" + ) if self is Cover.BOTH: - return "a deployed visual check and an evaluable gate; either alone would hold it" + return ( + "a deployed visual check and a gate that fires unattended; either alone would hold it" + ) return "nothing; the row states whether a detector or a different instrument would close it" @@ -181,9 +241,19 @@ class CoverageRow: vision_deployable: bool # exists AND every requirement is met in this workcell vision_reason: str gate: Optional[str] # the declared gate that would reveal it, if any - gate_evaluable: bool # exists AND its inputs arrive AND the assay suits the sample + gate_standing: GateStanding # what that gate does here: the six answers, not a boolean gate_reason: str + @property + def gate_evaluable(self) -> bool: + """Whether the gate counts as coverage. Derived, so it cannot drift from the standing. + + Kept as a name because it is what a caller asks, and kept as a property because the + two-valued answer is one reading of a six-valued fact and the six-valued fact is the + one this module stores. + """ + return self.gate_standing.counts_as_coverage + @property def severity(self) -> Severity: return self.failure.severity @@ -232,21 +302,39 @@ def destroys_material(self) -> bool: @property def residual(self) -> Optional[Detection]: - """The path `recovery` resolved that is neither a camera nor a gate, where one survives. + """The path `recovery` resolved that this module does not count, where one survives. - None when something machine-checkable already holds the failure, and None when recovery - found nothing either. Anything else is a real path this module deliberately does not count - as coverage, and reporting it is what keeps an uncovered row from being read as a claim + None when something already holds the failure, and None when recovery found nothing + either. Anything else is a real path this module deliberately does not count as + coverage, and reporting it is what keeps an uncovered row from being read as a claim that the failure is invisible to the entire lab. + + A path this row has already REFUSED is also None, and that is the difference between a + residual and an excuse. `recovery` resolves the declared path against its own inputs, + which are not always these: it takes a QC_GATE on trust when nobody handed it a gate + report, and it asks only whether a gate can fire, not whether the number is about this + sample. Republishing its answer beside a gate_reason that says the gate is unsatisfiable + would put two computations of one fact on one row, disagreeing, with the reader left to + pick. SUPERVISED survives here on purpose: that gate does reveal the failure, to a + person, which is precisely a path worth naming and not worth counting. """ if self.cover.covered or self.effective.silent: return None + if self.effective.detection is Detection.QC_GATE: + return self.effective.detection if self.gate_standing.reveals_the_failure else None + if self.effective.detection is Detection.VISION and not self.vision_deployable: + return None return self.effective.detection @property def unwatched(self) -> bool: - """No camera, no gate, and nothing else either. The rows that should be read first.""" - return not self.cover.covered and self.effective.silent + """No camera, no gate, and nothing else either. The rows that should be read first. + + Asked of `residual` rather than of `effective.silent` directly, because a failure whose + only remaining path this row just refused is unwatched in fact however loudly the plan + claims otherwise. + """ + return not self.cover.covered and self.residual is None def closes_it(self) -> str: """The next action that would put a machine check between this failure and the material. @@ -332,20 +420,42 @@ def unwatched(self) -> List[CoverageRow]: """No camera, no gate, and `recovery` found nothing else.""" return [r for r in self.rows if r.unwatched] + def destructive(self) -> List[CoverageRow]: + """Every row whose failure destroys material, held or not. What `complete` is about.""" + return [r for r in self.rows if r.destroys_material] + + def in_scope(self) -> bool: + """Whether this report contains anything for an all-clear to be an all-clear ABOUT. + + A report over rows none of which destroy material has no material-destroying failure + left uncovered, and answering "complete" to that is the vacuous pass wearing the last + costume this module had left. A row count does not test it: one row about a reader + timing out is enough to make an empty destructive population look like a cleared one, + and a caller scoping a report to a subprotocol gets an all-clear for a lab with no + machine check anywhere. + + Separated rather than folded into `complete` because "nothing here destroys material" + is a real and different verdict from "everything that does is held", the same way + `Uncovered` splits one empty cell into two findings that imply different budgets. + """ + return bool(self.destructive()) + def complete(self) -> bool: - """True only when every material-destroying failure has a machine check on it. + """True only when this report has material-destroying failures and all are held. - `bool(self.rows)` is load-bearing, not defensive. An empty report satisfies the - all-clear by vacuity, and a composition that answered "complete" for a protocol it - resolved no failures on would be the same bug this package chases everywhere else: a - check that passes because it was handed nothing. + Both halves are load-bearing. The second is the claim; the first is what keeps the + claim from being earned by an empty population, which is the same bug this package + chases everywhere else -- a check that passes because it was handed nothing. An empty + report fails the first half, and so does a report that resolved only failures which + cost time. """ - return bool(self.rows) and not self.uncovered_destructive() + return self.in_scope() and not self.uncovered_destructive() def counts(self) -> Dict[str, int]: return { "total": len(self.rows), "covered": len(self.covered()), + "destructive": len(self.destructive()), "vision": sum(1 for r in self.rows if r.cover is Cover.VISION), "gate": sum(1 for r in self.rows if r.cover is Cover.GATE), "both": sum(1 for r in self.rows if r.cover is Cover.BOTH), @@ -370,12 +480,28 @@ def _vision_arm( Deployability is asked of `VisionCapability.available`, which already refuses an impossible check. Recomputing that test here would be a second opinion on a question vision owns. + + Observability is recorded twice in this package -- on the VisualCheck and on the + FailureMode -- and this function is the only place the two records meet. `available` + consults the check's; every consumer downstream of this row consults the failure's. So a + disagreement is refused here rather than resolved, because resolving it would mean this + module picking a winner between two siblings that each own their half of the fact, and a + silent pick makes "can a camera ever see this" answerable two ways in one report. """ check = check_catching(failure.name) if check is None: if failure.observable.reachable_by_vision: return None, False, "the condition is imageable and no visual check is declared for it" return None, False, "no camera resolves this condition at any model quality" + if check.observable is not failure.observable: + raise ValueError( + f"vision and the failure model disagree about '{failure.name}': the check " + f"'{check.name}' records it as {check.observable.value} while the failure mode " + f"records it as {failure.observable.value}. Composing the two would let a deployed " + f"check close a failure this package calls physically unreachable, or hold a ceiling " + f"over one a camera sees. Correct it in vision or in recovery; coverage will not " + f"choose between them." + ) if capability.available(check): return check.name, True, f"'{check.name}' is deployed and validated in this workcell" if not check.possible: @@ -391,40 +517,85 @@ def _vision_arm( def _gate_arm( failure: FailureMode, gates: GateReport, measurements: Dict[str, Measurement] -) -> Tuple[Optional[str], bool, str]: - """(gate name, counts as coverage, reason) for the assay side of one failure. +) -> Tuple[Optional[str], GateStanding, str]: + """(gate name, standing, reason) for the assay side of one failure. - Three ways a declared gate fails to be coverage, in increasing order of how quietly it + Five ways a declared gate fails to be coverage, in increasing order of how quietly it does so. It may not be on this protocol at all. It may be UNSATISFIABLE, which at least - fails loudly -- no number arrives. Or it may be evaluable and resting on a measurement that - is the wrong assay for the sample, which is the one that spends the flow cell: the + fails loudly -- no number arrives. It may fire only with a human at the instrument, which + is a real check and is not one this workcell has when it runs the way it was built to run. + It may read a measurement nobody declared, in which case whether the assay suits the + sample is unknown and unknown is not a pass. Or it may be evaluable and resting on a + measurement that is the wrong assay, which is the one that spends the flow cell: the instrument returns a real, precise, reproducible number about something other than the library, and the gate passes on it. + + Every row bearing the name is resolved, not the first one. A report carrying two rows for + one gate is malformed, and the failure mode of `next(...)` over a malformed report is that + an optimistic duplicate placed above a dead one silences the dead one -- an author-settable + field that silences a report, by ordering rather than by value. The worst row wins, which + is also what `qc.GateReport.unsatisfiable` and `closes_the_loop` already say about the same + object: a gate that is unsatisfiable in any row of the report is unsatisfiable. """ name = failure.via_gate if name is None: - return None, False, "no gate on this protocol reads what this failure changes" - row = next((r for r in gates.rows if r.gate.name == name), None) - if row is None: + return ( + None, + GateStanding.ABSENT, + "no gate on this protocol reads what this failure changes", + ) + matches = [r for r in gates.rows if r.gate.name == name] + if not matches: # The name is reported in the reason and deliberately NOT returned as the gate. A gate # that is not on this protocol reads nothing here, and returning its name would let the # uncovered classification treat a dangling reference as an assay that exists. return ( None, - False, + GateStanding.ABSENT, f"the plan names '{name}' for this failure, and no such gate is on this protocol", ) - if not row.evaluable: - return name, False, f"'{name}' is unsatisfiable: {row.reason}" + dead = next((r for r in matches if not r.evaluable), None) + if dead is not None: + return name, GateStanding.DEAD, f"'{name}' is unsatisfiable: {dead.reason}" + attended = next((r for r in matches if r.readiness is not Readiness.READY), None) + if attended is not None: + return ( + name, + GateStanding.SUPERVISED, + f"'{name}' can be evaluated only with a human at the instrument, which is not coverage " + f"in a workcell that runs unattended: {attended.reason}", + ) + undeclared: List[str] = [] + for row in matches: + for key in row.gate.measurements: + if key not in measurements and key not in undeclared: + undeclared.append(key) + if undeclared: + # `qc.GateReport.inappropriate` skips a key it cannot resolve, which is correct for the + # question it answers -- it lists the assays known to be wrong -- and is fail-open for + # the question asked here. Silence about an assay is not a statement that the assay + # suits the sample, and converting it into one is the same bug as a gate evaluated + # against an empty measurement dict passing every criterion it was never handed. + return ( + name, + GateStanding.UNMEASURED, + f"'{name}' reads '{undeclared[0]}', which is not a declared measurement here, so " + f"whether that assay suits this sample is unknown -- an unchecked assay is not an " + f"appropriate one", + ) wrong = [m for gate_name, m in gates.inappropriate(measurements) if gate_name == name] if wrong: return ( name, - False, + GateStanding.WRONG_ASSAY, f"'{name}' can be evaluated and rests on '{wrong[0].key}', which is the wrong assay " f"for this sample: {wrong[0].inappropriate_reason}", ) - return name, True, f"'{name}' is evaluable and reads an assay appropriate to the sample" + return ( + name, + GateStanding.COUNTS, + f"'{name}' is evaluable and reads an assay appropriate to the sample", + ) def coverage_report( @@ -441,12 +612,20 @@ def coverage_report( re-derived: the failure list comes from `recovery`, gate readiness from `qc`, and deployability from `vision`. That is why this cannot disagree with any of them. - It refuses two kinds of incoherent input rather than averaging over them. Reports about + It refuses three kinds of incoherent input rather than averaging over them. Reports about different protocols compose into a number about no lab at all. A recovery report built against a different vision capability than the one passed here does the same thing more quietly, because every row still looks well-formed -- so the mismatch is checked against the one place the two are forced to agree: a failure whose declared path is a camera is resolved by `recovery` exactly when its check is deployable. + + The gate axis is checked the same way and for the same reason. `recovery_report(protocol)` + with no gate report is a valid public call and takes every declared QC_GATE path on trust, + so composing one with a real gate report produces rows that assert a gate is dead and that + the declared path still stands on it, in the same row -- and `unwatched()`, the list a + reader is told to read first, quietly loses the failures that matter most. Guarding one + axis and not the other is worse than guarding neither, because the guarded one makes the + gap look deliberate. """ cap = vision_capability or VisionCapability.none() meas = measurements if measurements is not None else MEASUREMENTS @@ -468,7 +647,7 @@ def coverage_report( for eff in recovery.rows: failure = eff.failure check_name, deployable, vision_reason = _vision_arm(failure, cap) - gate_name, evaluable, gate_reason = _gate_arm(failure, gates, meas) + gate_name, standing, gate_reason = _gate_arm(failure, gates, meas) if failure.declared_detection is Detection.VISION: if (eff.detection is Detection.VISION) != deployable: raise ValueError( @@ -479,6 +658,22 @@ def coverage_report( f"built against a different capability, and composing the two would report a " f"workcell nobody has." ) + if failure.declared_detection is Detection.QC_GATE: + # Compared against the gate report's own evaluability and not against `standing`. + # This arm additionally refuses a supervised gate and an assay that does not suit the + # sample, questions `recovery.effective` never asks, so comparing to the standing + # would raise on the library_conc_od case where the two layers agree exactly. + declared = [r for r in gates.rows if r.gate.name == failure.via_gate] + live = bool(declared) and all(r.evaluable for r in declared) + if (eff.detection is Detection.QC_GATE) != live: + raise ValueError( + f"the recovery report and the gate report disagree about '{failure.name}': " + f"recovery resolved it to '{eff.detection.value}' while the gate report for " + f"'{gates.protocol}' says '{failure.via_gate}' is " + f"{'evaluable' if live else 'not evaluable'}. The recovery report was built " + f"against different gates, or against none at all, and composing the two would " + f"put a dead gate and a live detection path on one row." + ) rows.append( CoverageRow( failure=failure, @@ -487,7 +682,7 @@ def coverage_report( vision_deployable=deployable, vision_reason=vision_reason, gate=gate_name, - gate_evaluable=evaluable, + gate_standing=standing, gate_reason=gate_reason, ) ) @@ -527,13 +722,38 @@ def destroys_material(self) -> bool: return _destroys_material(self.failure) +@dataclass(frozen=True) +class StillBlocked: + """One failure a camera could reach that THIS proposal does not reach. + + Neither a lift nor a ceiling, and the row that decides whether a partial purchase is worth + making. The ceiling says a failure is out of reach at any price. This says it is in reach + and that the money on the table does not buy it, and it names the requirement that would. + """ + + failure: FailureMode + check: Optional[str] + missing: Tuple[VisionRequirement, ...] # what the proposal still lacks for this check + reason: str + + @property + def destroys_material(self) -> bool: + return _destroys_material(self.failure) + + @dataclass class SotaLift: - """What a proposed vision capability buys, and what it provably does not. - - Both lists are required output. A report that showed only `lifted` would be a purchase - justification, and every purchase justification for a better model is correct about the - failures it lists and silent about the ones that make the purchase insufficient. + """What a proposed vision capability buys, what it provably cannot, and what it misses. + + All three lists are required output. A report that showed only `lifted` would be a + purchase justification, and every purchase justification for a better model is correct + about the failures it lists and silent about the ones that make the purchase + insufficient. `ceiling` alone does not fix that, because the ceiling only holds the + failures no capability reaches: a real proposal is partial, and the failures it leaves + uncovered while a further purchase would close them fall between the two lists and + vanish. `still_blocked` is where they land, and for the single-cell loop it is where the + most destructive routine failure in low-input prep lands under any proposal missing + LIGHTING. """ protocol: str @@ -541,6 +761,7 @@ class SotaLift: proposed: VisionCapability lifted: List[Lift] = field(default_factory=list) ceiling: List[Ceiling] = field(default_factory=list) + still_blocked: List[StillBlocked] = field(default_factory=list) @property def buys_nothing(self) -> bool: @@ -550,12 +771,18 @@ def destructive_ceiling(self) -> List[Ceiling]: """The ceiling rows that cost material. The sentence a budget meeting needs.""" return [c for c in self.ceiling if c.destroys_material] + def destructive_still_blocked(self) -> List[StillBlocked]: + """The still-blocked rows that cost material. The sentence that prices the next step.""" + return [b for b in self.still_blocked if b.destroys_material] + def counts(self) -> Dict[str, int]: return { "lifted": len(self.lifted), "lifted_destructive": sum(1 for lift in self.lifted if lift.destroys_material), "ceiling": len(self.ceiling), "ceiling_destructive": len(self.destructive_ceiling()), + "still_blocked": len(self.still_blocked), + "still_blocked_destructive": len(self.destructive_still_blocked()), } @@ -573,7 +800,14 @@ def sota_lift( capability, because gates do not move when a model does. What changes between them is what the capability buys. What is uncovered and INVISIBLE in the second is the ceiling, and no further capability moves it -- so a lab reading only the lift will keep spending against - failures that are not a model problem. + failures that are not a model problem. What is uncovered and still imageable in the second + is what the proposal misses, and a lab reading only the first two lists will believe a + partial purchase was a complete one. + + Every uncovered row of the second report lands in the ceiling or in still_blocked, and the + partition is checked rather than assumed. A row that falls out of both is not an omission a + reader can see: the counts still add up, the lists still read cleanly, and the failure is + simply absent. """ led = ledger if ledger is not None else build_ledger(protocol) gts = gates if gates is not None else gate_report(protocol.name, led) @@ -590,15 +824,17 @@ def sota_lift( lifted: List[Lift] = [] ceiling: List[Ceiling] = [] + still_blocked: List[StillBlocked] = [] for row in after.rows: was = prior.get(row.failure.name) moved = was is not None and not was.cover.covered and row.cover.covered # A lift has to be attributable to a deployed check. The gate report is shared between - # the two runs, so nothing but vision can move a row -- and the INVISIBLE test is - # unreachable by construction, since VisionCapability.available already refuses an - # impossible check. Both are checked anyway: a lift reported for an INVISIBLE failure is - # the exact overclaim this function exists to prevent, and a guard that never fires costs - # nothing next to one that fires once against a real purchase. + # the two runs, so nothing but vision can move a row -- and the INVISIBLE test cannot + # fire, because `_vision_arm` refuses a check whose observability disagrees with the + # failure's and `VisionCapability.available` refuses an impossible check. Both are + # checked anyway: a lift reported for an INVISIBLE failure is the exact overclaim this + # function exists to prevent, and a guard that never fires costs nothing next to one + # that fires once against a real purchase. attributable = ( row.observable.reachable_by_vision and row.vision_deployable @@ -621,13 +857,15 @@ def sota_lift( ) ) continue - if not row.cover.covered and row.observable is Observable.INVISIBLE: + if row.cover.covered: + continue + if row.observable is Observable.INVISIBLE: # The residual is named where one survives, so that a row like star_resource_busy -- # invisible, and loudly raised by the instrument itself -- is not read as a hole # merely because a camera cannot reach it. The ceiling is a statement about vision, # not a claim that nothing in the lab notices. residual = row.residual - standing = ( + carried = ( f" The declared {residual.value} path still stands on it at " f"{row.effective.latency.value}." if residual is not None @@ -639,19 +877,68 @@ def sota_lift( reason=( f"'{row.failure.name}' is INVISIBLE: the failed run and the clean run produce " f"identical images, so no vision capability reaches it at any model quality. " - f"{row.gate_reason}.{standing}" + f"{row.gate_reason}.{carried}" ), ) ) + continue + check = check_catching(row.failure.name) + missing = proposed_capability.missing_for(check) if check is not None else () + if check is None: + reason = ( + f"'{row.failure.name}' is imageable and no visual check is declared for it, so this " + f"proposal cannot reach it at any capability until one is written" + ) + elif missing: + reason = ( + f"'{row.failure.name}' is imageable and '{check.name}' would hold it, and " + f"'{proposed_capability.name}' still misses " + + ", ".join(m.value for m in missing) + + ". This is a purchase away, not a physics problem, and the proposal on the table " + "does not make it" + ) + else: + # Unreachable while `available` is a subset test over exactly `missing_for`. Kept + # because the alternative is a StillBlocked row whose reason column is empty, which + # reads as a failure nobody could account for. + reason = ( + f"'{row.failure.name}' is imageable, '{check.name}' is deployable under this " + f"proposal, and the row is still uncovered; this module cannot say why" + ) + still_blocked.append( + StillBlocked( + failure=row.failure, + check=row.vision_check, + missing=missing, + reason=reason, + ) + ) + + reported = {c.failure.name for c in ceiling} | {b.failure.name for b in still_blocked} + remaining = {row.failure.name for row in after.uncovered()} + if reported != remaining: + # The three lists are the whole product, so a row that reaches none of them is a failure + # this function computed and then dropped. Refusing is the only honest response: a + # SotaLift missing a row is indistinguishable, to its reader, from one where the + # proposal covered it. + raise ValueError( + "sota_lift resolved failures it then reported in no list: " + + ", ".join(sorted(remaining - reported)) + + ". Every uncovered failure under the proposed capability belongs to the ceiling or " + "to still_blocked, and a report that drops one is the purchase justification the " + "second and third lists exist to prevent." + ) lifted.sort(key=lambda lift: (not lift.destroys_material, lift.failure.name)) ceiling.sort(key=lambda c: (not c.destroys_material, c.failure.name)) + still_blocked.sort(key=lambda b: (not b.destroys_material, b.failure.name)) return SotaLift( protocol=protocol.name, current=current_capability, proposed=proposed_capability, lifted=lifted, ceiling=ceiling, + still_blocked=still_blocked, ) @@ -720,8 +1007,8 @@ def mandatory_gates( ) ) continue - name, evaluable, reason = _gate_arm(failure, gates, meas) - out.append(Demand(failure=failure, gate=name, met=evaluable, reason=reason)) + name, standing, reason = _gate_arm(failure, gates, meas) + out.append(Demand(failure=failure, gate=name, met=standing.counts_as_coverage, reason=reason)) return tuple(out) @@ -739,8 +1026,10 @@ def unmet_demands( "CoverageReport", "CoverageRow", "Demand", + "GateStanding", "Lift", "SotaLift", + "StillBlocked", "Uncovered", "coverage_report", "mandatory_gates", diff --git a/autonomous_lab/durability.py b/autonomous_lab/durability.py index 077d9ee..160cb1a 100644 --- a/autonomous_lab/durability.py +++ b/autonomous_lab/durability.py @@ -36,6 +36,15 @@ `intelligence.trusted_for` applies to an unbenchmarked operation and `qc.evaluate` applies to an absent measurement. +That ordering ranks verdicts and does not decide them, and one asymmetry falls out of it. +LAPSED is a proof and UNKNOWN is the absence of one, so an unmeasured unit cannot cancel a +lapse already proved on a unit that WAS counted -- it is reported alongside as an +additional finding on the same obligation. The reverse does not hold at all: ENTITLED and +EXPIRING are clearances, and nothing unmeasured supports a clearance. An interval charged +in days and runs, five hundred days past due, against a lab that never counted runs, is +lapsed on the evidence in hand, and reporting it merely UNKNOWN dropped the retrospective +alarm this module calls the harder problem. + AN INTERVAL CARRIES ITS BASIS. A vendor-stated service interval is real evidence and it is not evidence about THIS duty cycle, which is the identical distinction `qc.Basis` already draws about thresholds -- so this module reuses that enum rather than mirroring it. A @@ -55,6 +64,7 @@ from __future__ import annotations +import math from dataclasses import dataclass from enum import Enum from typing import Dict, List, Optional, Tuple @@ -175,7 +185,14 @@ class Interval: `restores` is prose and deliberately concrete, in the same discipline `provenance.CustodyGap.closes_it` applies: "improve maintenance" is not an action, "run - the two-point absorbance calibration and record both readings" is. + the two-point absorbance calibration and record both readings" is. It is required to say + something, because `untrusted_instruments` reports an instrument's remedies by + collecting this field, and a blank one produces a row that names an instrument untrusted + and names nothing that would fix it. + + `instrument` is checked against the health record it is resolved through, not decoration. + See `InstrumentHealth.__post_init__`: the same fact stored in two places disagrees + eventually, and the disagreement here reports the wrong box entitled. """ name: str @@ -189,6 +206,14 @@ class Interval: rationale: str = "" def __post_init__(self) -> None: + if not self.restores.strip(): + # An obligation that does not say what discharging it restores is an author-settable + # field that silences half the report: the instrument is still named untrusted and + # the work that would restore it is gone, which is the one thing the row was for. + raise ValueError( + f"interval '{self.name}' does not say what discharging it restores; an obligation " + "with no stated remedy is a note, not an obligation" + ) declared = { Charge.DAYS: self.every_days, Charge.RUNS: self.every_runs, @@ -203,7 +228,20 @@ def __post_init__(self) -> None: "not an obligation, it is a note" ) for charge, value in declared.items(): - if value is not None and value <= 0: + if value is None: + continue + if not math.isfinite(value): + # The same never-comes-due obligation written as a number. An infinite recurrence + # is never exhausted and never crossed, and a NaN one compares False against every + # test in this module, so both report entitled forever through the arithmetic + # instead of through the missing field the guard above catches. Checked before the + # sign test, which NaN passes. + raise ValueError( + f"interval '{self.name}' recurs every {value} {charge.value}, which is not a " + "quantity that ever arrives; an obligation that never comes due is not an " + "obligation, however it is spelled" + ) + if value <= 0: raise ValueError( f"interval '{self.name}' recurs every {value} {charge.value}, which is already past due " "the moment it is discharged" @@ -248,6 +286,15 @@ class Accrued: entitlement to an instrument nobody has been watching, which is the failure this module exists to prevent. + None is the only sentinel for uncounted, and NaN is refused rather than accepted as a + second one. That refusal is the whole promise above, held against the way the number + actually arrives: a blank maintenance cell round-tripped through a spreadsheet, a CSV, + or a JSON reader comes back as NaN, which is precisely the "nobody counted it" case this + module exists for. NaN compares False against every threshold here, so accepting it + reports a full budget and an entitled instrument over a cell nobody filled in. Refusing + at the door tells the loader what to do -- an unfilled cell has to arrive as None -- + rather than guessing which of the two it meant. + Structurally identical to `Campaign` and kept separate anyway, because one is a count of the past and the other is a claim about the future, and a function that accepts either where it meant one will eventually be handed the wrong one. @@ -257,6 +304,25 @@ class Accrued: runs: Optional[float] = None hours: Optional[float] = None + def __post_init__(self) -> None: + for charge in Charge: + value = self.charged(charge) + if value is None: + continue + if not math.isfinite(value): + raise ValueError( + f"{value} {charge.value} accrued is not a count. A blank maintenance cell " + "round-tripped through a spreadsheet arrives as NaN and means UNCOUNTED, so it " + "has to arrive as None; read as a measurement it restores entitlement to an " + "instrument nobody was watching" + ) + if value < 0: + raise ValueError( + f"{value} {charge.value} accrued since this obligation was discharged; an " + "instrument cannot have done negative work since it was serviced, and a " + "negative accrual mints budget that was never earned" + ) + def charged(self, charge: Charge) -> Optional[float]: if charge is Charge.DAYS: return self.days @@ -297,12 +363,32 @@ class Campaign: ninety days charges ninety days against the calibration of every instrument in it, including the ones it never touches, and a plan that counts only its own plates will miss the boundary that catches it. + + Zero is the default and it means no work declared in that unit, which is why the + all-zero campaign is refused where it is used rather than where it is built -- see + `_refuse_empty_campaign`. Negative and non-finite quantities are refused here, at the + door: a plan of minus five runs is not empty, so it survives that refusal, and it can + never cross a boundary because it consumes budget instead of spending it. """ runs: float = 0.0 hours: float = 0.0 days: float = 0.0 + def __post_init__(self) -> None: + for charge in Charge: + value = self.charged(charge) + if not math.isfinite(value): + raise ValueError( + f"a campaign of {value} {charge.value} is not a plan; a quantity that compares " + "False against every boundary in this module clears every one of them" + ) + if value < 0: + raise ValueError( + f"a campaign of {value} {charge.value} is not a plan; planned work is what will " + "be charged against an instrument's budget, and nothing gives budget back" + ) + def charged(self, charge: Charge) -> float: if charge is Charge.DAYS: return self.days @@ -343,12 +429,25 @@ def value(self, charge: Charge) -> Optional[float]: return v return None + @staticmethod + def _spent(left: float) -> bool: + """The boundary convention, written once because two copies of it drift apart. + + A budget of exactly zero is spent, not nearly spent. `exhausted` asks this of the + present and `crossed_by` asks it of the state the plan ends in, and while the two were + written separately they disagreed: a campaign consuming exactly the remaining budget + was reported clear, and the state that same campaign produces is one this module calls + lapsed. The last plate of a "clear" campaign was run by an instrument this module + would not vouch for. + """ + return left <= 0 + def exhausted(self) -> Tuple[Charge, ...]: """Units whose budget is spent. A budget of exactly zero is spent, not nearly spent.""" - return tuple(c for c, v in self.left if v <= 0) + return tuple(c for c, v in self.left if self._spent(v)) def crossed_by(self, campaign: Campaign) -> Tuple[Charge, ...]: - """Units where the planned work consumes more than is left. + """Units where the planned work ends on or past the boundary. Excludes units already exhausted. Those are a lapse, which is a different finding with a different fix, and folding them in here would let a report about a plan absorb a @@ -356,7 +455,7 @@ def crossed_by(self, campaign: Campaign) -> Tuple[Charge, ...]: """ out: List[Charge] = [] for c, v in self.left: - if v > 0 and campaign.charged(c) > v: + if not self._spent(v) and self._spent(v - campaign.charged(c)): out.append(c) return tuple(out) @@ -369,6 +468,14 @@ class Standing: lapse is restored by performing the obligation, an unrecorded one by performing it AND recording it, and a crossing by discharging the obligation before the campaign starts rather than during it. + + `partial` names the units nobody counted on an obligation where other units WERE + counted, which is a different state from an obligation nobody counted at all and used to + be reported as the same one. It qualifies a finding and never suppresses it: an interval + charged in days and runs, past due on days, against a lab that never counted runs, is + lapsed on the evidence it has -- and reporting that UNKNOWN because of the second unit + dropped the row out of `lapsed()` and `invalidating()` and took the retrospective alarm + with it. """ instrument: str @@ -378,6 +485,7 @@ class Standing: reason: str restores: str crossing: Tuple[Charge, ...] = () + partial: Tuple[Charge, ...] = () @property def trustworthy(self) -> bool: @@ -405,6 +513,26 @@ def _remaining(interval: Interval, record: Optional[ServiceRecord]) -> Remaining ) +def _refuse_empty_campaign(campaign: Optional[Campaign]) -> None: + """The empty-campaign refusal, called from every door that accepts a plan. + + It lived on `crosses_boundary` alone, which is a guard on one of four entry points and + therefore a guard on none: the same empty Campaign passed through `standing`, + `InstrumentHealth.entitlement`, `entitlement_summary` and `untrusted_instruments` and + cleared every crossing check in all four, because zero planned work crosses nothing. A + planner who forgot to fill the campaign in got a workcell-wide clean bill. + + `campaign=None` is the no-plan case and stays legal. It is a different claim from a + campaign of zero: one asks what the instrument is entitled to today, the other asks + whether a plan fits, and the plan is missing. + """ + if campaign is not None and campaign.empty: + raise ValueError( + "a campaign that plans no runs, hours, or days crosses nothing; state the planned " + "work before asking whether it fits inside the instrument's entitlement" + ) + + def standing( health: "InstrumentHealth", interval: Interval, @@ -416,7 +544,23 @@ def standing( record is UNKNOWN before anything else is considered, because there is no baseline to subtract from; an obligation whose record is only ASSERTED is UNKNOWN for the same reason `provenance` refuses to call a command log a record of events. + + A proved lapse is checked BEFORE an uncounted unit, and the asymmetry is deliberate. + LAPSED is a proof and UNKNOWN is the absence of one, so an unmeasured second unit cannot + make a past-due first unit less past due -- it is reported alongside, in `partial`. The + reverse does not hold: ENTITLED and EXPIRING are clearances, and an uncounted unit + cannot support a clearance, so those two stay behind the uncounted check. """ + _refuse_empty_campaign(campaign) + if interval.instrument != health.instrument: + # The obligation and the health record each name an instrument, and nothing used to + # compare them. Resolving one box's paperwork against another's service history + # returns a verdict about neither, and it returns it as a PASS rather than as a gap. + raise ValueError( + f"obligation '{interval.name}' belongs to '{interval.instrument}' and is being " + f"resolved against the service history of '{health.instrument}'; one box's " + "paperwork does not entitle another" + ) record = health.record_for(interval.name) remaining = _remaining(interval, record) @@ -436,36 +580,39 @@ def standing( ), ) - if not record.attestation.is_evidence: + clash = health.contradictions(interval.name) + if clash: + moments = ", ".join(sorted({r.performed for r in clash})) return Standing( instrument=health.instrument, interval=interval, entitlement=Entitlement.UNKNOWN, remaining=remaining, reason=( - f"'{interval.name}' is recorded as {record.attestation.value}: software marked it " - "done and nothing read back. That is intent, not evidence that it happened" + f"{len(clash)} records claim '{interval.name}' was discharged on {moments} and " + "disagree about what has accrued since. Which one governs is decided by which row " + "was appended last, so the same facts return entitled or lapsed by declaration " + "order; an instrument with two incompatible histories is unmeasured" ), restores=( - "have the person or instrument that performed it attest to it, so the record is " - f"witnessed or confirmed rather than asserted; otherwise {interval.restores}" + f"withdraw or correct the duplicate records of '{interval.name}' on {moments} so " + "one discharge is claimed once, with the accrual somebody actually counted" ), ) - if remaining.uncounted: - units = ", ".join(c.value for c in remaining.uncounted) + if not record.attestation.is_evidence: return Standing( instrument=health.instrument, interval=interval, entitlement=Entitlement.UNKNOWN, remaining=remaining, reason=( - f"'{interval.name}' is charged in {units}, and nobody counted {units} since it was " - f"last discharged on {record.performed}. There is no budget to state" + f"'{interval.name}' is recorded as {record.attestation.value}: software marked it " + "done and nothing read back. That is intent, not evidence that it happened" ), restores=( - f"count {units} against this instrument from now on; until then the interval is a " - "policy nobody is measuring against" + "have the person or instrument that performed it attest to it, so the record is " + f"witnessed or confirmed rather than asserted; otherwise {interval.restores}" ), ) @@ -478,13 +625,56 @@ def standing( if interval.kind.invalidates_past_results else "" ) + # The uncounted units qualify this finding and do not replace it. Reporting UNKNOWN + # here because a second unit went uncounted erased a lapse the arithmetic had already + # proved: the row left `lapsed()` and `invalidating()`, the retrospective alarm went + # quiet, and the lab was told to start a counter instead of being told that months of + # results are unattributable. + unmeasured = "" + also = "" + if remaining.uncounted: + names = ", ".join(c.value for c in remaining.uncounted) + unmeasured = ( + f" Nobody counted {names} since, so this obligation is past due on what was " + "counted and unmeasured on the rest, which can only make it worse." + ) + also = f", and count {names} against this instrument from now on" return Standing( instrument=health.instrument, interval=interval, entitlement=Entitlement.LAPSED, remaining=remaining, - reason=f"'{interval.name}' is past due on {units}.{past}", - restores=interval.restores, + reason=f"'{interval.name}' is past due on {units}.{past}{unmeasured}", + restores=f"{interval.restores}{also}", + partial=remaining.uncounted, + ) + + if remaining.uncounted: + units = ", ".join(c.value for c in remaining.uncounted) + # Never claim there is no budget while `remaining.left` holds one. What cannot be + # stated is the budget for the interval, because it comes due on whichever unit + # arrives first and one of those units is unmeasured. + stated = ", ".join(f"{v} {c.value}" for c, v in remaining.left) + budget = ( + f" {stated} remain on the units that were counted, and the obligation comes due on " + "whichever unit arrives first, so that is not this interval's budget" + if remaining.left + else " There is no budget to state" + ) + return Standing( + instrument=health.instrument, + interval=interval, + entitlement=Entitlement.UNKNOWN, + remaining=remaining, + reason=( + f"'{interval.name}' is charged in {units}, and nobody counted {units} since it was " + f"last discharged on {record.performed}.{budget}" + ), + restores=( + f"count {units} against this instrument from now on; until then the interval is a " + "policy nobody is measuring against" + ), + partial=remaining.uncounted if remaining.left else (), ) if campaign is not None: @@ -533,8 +723,16 @@ class InstrumentHealth: An instrument with no declared obligations is UNKNOWN rather than entitled. Declaring none is not the same as having none, and treating an empty obligation list as a clean - bill of health is exactly the vacuous pass a gate evaluated against an empty measurement - dict gives. + bill of health is exactly the vacuous pass `qc.evaluate` gives a gate that declares no + criteria -- which `qc.readiness` refuses and `qc.evaluate` does not. The analogue is the + empty CRITERIA tuple and deliberately not the empty measurement dict: `qc.evaluate` + handles that one correctly, and citing the case a sibling gets right as the precedent + for a case it gets wrong is the kind of claim this package's own data contradicts. + + `instrument` is the owning key, and every obligation filed here has to agree with it. + The name is stored in two places -- here and on `Interval` -- and two stored copies of + one fact eventually disagree, so the disagreement is refused at construction rather than + discovered later as an instrument reporting entitled on another box's paperwork. """ instrument: str @@ -542,6 +740,27 @@ class InstrumentHealth: records: Tuple[ServiceRecord, ...] = () note: str = "" + def __post_init__(self) -> None: + seen: List[str] = [] + for iv in self.obligations: + if iv.instrument != self.instrument: + raise ValueError( + f"obligation '{iv.name}' declares instrument '{iv.instrument}' and is filed " + f"under '{self.instrument}'; a maintenance plan for one box does not entitle " + "another, and the misfiling reports a pass rather than a gap" + ) + if iv.name in seen: + # A service record names the interval it discharged and nothing else. Two + # obligations sharing a name are therefore discharged by one signature, so a + # calibration and a qualification that were never both performed both read as + # done. 'annual_calibration', 'pm' and 'tip_change' are exactly the names a real + # lab reuses. + raise ValueError( + f"'{iv.name}' is declared twice on {self.instrument}; two obligations sharing a " + "name are discharged by one record, for work that was performed once" + ) + seen.append(iv.name) + @classmethod def unrecorded(cls, instrument: str) -> "InstrumentHealth": """The honest zero state: an instrument nobody has written anything down about.""" @@ -553,11 +772,17 @@ def unrecorded(cls, instrument: str) -> "InstrumentHealth": ), ) + def records_for(self, interval_name: str) -> Tuple[ServiceRecord, ...]: + """Every record filed against one obligation, in the order the caller declared them.""" + return tuple(r for r in self.records if r.interval == interval_name) + def record_for(self, interval_name: str) -> Optional[ServiceRecord]: """The most recent record for an obligation, taking the last declared as most recent. Order is the caller's, not a sort. Sorting would need dates parsed out of `performed`, - and this module does no date arithmetic on purpose. + and this module does no date arithmetic on purpose. Records of separate discharges are + a service history and the later supersedes; records of the SAME discharge that + disagree are not a history, and `contradictions` catches those before this is read. """ found: Optional[ServiceRecord] = None for rec in self.records: @@ -565,7 +790,30 @@ def record_for(self, interval_name: str) -> Optional[ServiceRecord]: found = rec return found + def contradictions(self, interval_name: str) -> Tuple[ServiceRecord, ...]: + """Records that claim the SAME discharge of an obligation and disagree about it. + + Two records performed at different moments are a history, and taking the last is what + `record_for` is for. Two records claiming one discharge with different accruals or + different attestations are a contradiction, and resolving it by tuple position decides + the verdict by declaration order: the same instrument reads entitled or lapsed + depending on which row was appended last, so appending a flattering one buries a + lapse. Reported rather than resolved -- an instrument with two incompatible histories + is unmeasured, which this module already ranks below overdue. + """ + by_moment: Dict[str, List[ServiceRecord]] = {} + for rec in self.records_for(interval_name): + by_moment.setdefault(rec.performed, []).append(rec) + out: List[ServiceRecord] = [] + for group in by_moment.values(): + # `since` and `attestation` are the two fields a verdict is computed from. Two + # technicians signing one job is not a contradiction about the budget. + if len({(r.since, r.attestation) for r in group}) > 1: + out.extend(group) + return tuple(out) + def standings(self, campaign: Optional[Campaign] = None) -> Tuple[Standing, ...]: + _refuse_empty_campaign(campaign) return tuple(standing(self, i, campaign) for i in self.obligations) def entitlement(self, campaign: Optional[Campaign] = None) -> Entitlement: @@ -575,6 +823,9 @@ def entitlement(self, campaign: Optional[Campaign] = None) -> Entitlement: that fails safe: a new instrument arrives untrusted and has to earn its way out, rather than arriving perfect because nobody has written its maintenance plan yet. """ + # Before the short circuit below, so an instrument with no obligations does not become + # the one door where an empty campaign still gets an answer. + _refuse_empty_campaign(campaign) if not self.obligations: return Entitlement.UNKNOWN worst = Entitlement.ENTITLED @@ -587,9 +838,23 @@ def lapsed(self) -> Tuple[Standing, ...]: return tuple(r for r in self.standings() if r.entitlement is Entitlement.LAPSED) def unrecorded_obligations(self) -> Tuple[Standing, ...]: - """Obligations this lab cannot say anything about, for any of the three reasons.""" + """Obligations this lab cannot say anything about, for any of the reasons.""" return tuple(r for r in self.standings() if r.entitlement is Entitlement.UNKNOWN) + def partially_counted(self) -> Tuple[Standing, ...]: + """Obligations resolved on some units while others went uncounted. + + Reported separately because the verdict on such a row rests on part of the evidence. + A lapse proved on the counted unit stands, and the uncounted unit is still an + unmeasured dimension on the same instrument -- two findings on one obligation, and + collapsing them to either one loses work. + """ + return tuple(r for r in self.standings() if r.partial) + + def conflicting(self) -> Tuple[Standing, ...]: + """Obligations whose records contradict each other rather than accumulate.""" + return tuple(r for r in self.standings() if self.contradictions(r.interval.name)) + def unvalidated(self) -> Tuple[Interval, ...]: """Obligations whose recurrence this lab has not confirmed against its own duty cycle. @@ -636,8 +901,15 @@ def uncertain(self) -> Tuple[Standing, ...]: Reported alongside real crossings rather than under them. An uncounted budget is not a small risk, it is an unavailable answer, and a campaign planned over one is planned over nothing. + + Includes rows whose verdict came from the units that WERE counted. A lapse proved on + one unit belongs in `lapsed()`, and the unit nobody counted on that same obligation is + still an unavailable answer -- one row appears in both lists because it is two + findings. """ - return tuple(r for r in self.rows if r.entitlement is Entitlement.UNKNOWN) + return tuple( + r for r in self.rows if r.entitlement is Entitlement.UNKNOWN or r.partial + ) @property def clear(self) -> bool: @@ -681,13 +953,11 @@ def crosses_boundary( A campaign that plans no work is refused rather than reported clear. Asking whether zero plates cross a boundary is a question about nothing, and returning a clean report to it - would be the emptiest kind of pass this package refuses. + would be the emptiest kind of pass this package refuses. The refusal itself lives in + `_refuse_empty_campaign` and every entry point that accepts a plan calls it, because + while it lived here the other three answered the same empty campaign with a clean bill. """ - if campaign.empty: - raise ValueError( - "a campaign that plans no runs, hours, or days crosses nothing; state the planned " - "work before asking whether it fits inside the instrument's entitlement" - ) + _refuse_empty_campaign(campaign) return BoundaryReport( instrument=health.instrument, campaign=campaign, @@ -714,6 +984,37 @@ class Untrusted: restores: Tuple[str, ...] +def _instrument_keys(workcell) -> List[str]: + """Every box this report is answerable for: present, plus federated, in that order.""" + keys: List[str] = list(workcell.present_keys()) + for key in workcell.federated: + if key not in keys: + keys.append(key) + return keys + + +def _health_at(key: str, supplied: Dict[str, InstrumentHealth]) -> InstrumentHealth: + """The health record for one instrument, refusing one filed under another's key. + + A missing record resolves to `unrecorded`: an instrument nobody wrote down is the case + this module exists for, and it reports UNKNOWN. A record filed under the wrong key is a + different thing entirely -- it is a record of some other box, and honoring it reports + this one entitled on service it never had. One record filed under nine keys reported a + whole workcell entitled and left nothing untrusted, off one calibration performed once + on one instrument. + """ + h = supplied.get(key) + if h is None: + return health_for(key) + if h.instrument != key: + raise ValueError( + f"health record for '{h.instrument}' is filed under '{key}'; a service history " + "belongs to the box the work was performed on, and crediting it to another reports " + "an instrument entitled on paperwork that is not its own" + ) + return h + + def untrusted_instruments( workcell, health: Optional[Dict[str, InstrumentHealth]] = None, @@ -731,16 +1032,35 @@ def untrusted_instruments( A missing health record is not a missing row. It resolves to `InstrumentHealth.unrecorded` and reports UNKNOWN, in the same way `throughput.duration_for` resolves an untimed op to UNKNOWN rather than to zero. + + A workcell that declares no instrument returns a row rather than an empty list. The + natural call shape is `if not untrusted_instruments(wc)`, which reads an empty list as a + lab where every instrument is entitled -- the same `all()`-over-nothing that + `BoundaryReport.clear` refuses. Emptiness is the finding here, not the absence of one. """ supplied = health or {} - keys: List[str] = list(workcell.present_keys()) - for key in workcell.federated: - if key not in keys: - keys.append(key) + _refuse_empty_campaign(campaign) + keys = _instrument_keys(workcell) + if not keys: + return [ + Untrusted( + instrument=workcell.name, + entitlement=Entitlement.UNKNOWN, + reason=( + f"workcell '{workcell.name}' declares no present or federated instrument, so " + "this report covers nothing; a lab with no boxes in it has not been shown to " + "have no boxes out of service" + ), + restores=( + "declare the instruments this workcell actually contains, then this report can " + "say something about them", + ), + ) + ] out: List[Untrusted] = [] for key in keys: - h = supplied.get(key) or health_for(key) + h = _health_at(key, supplied) verdict = h.entitlement(campaign) if verdict.trustworthy: continue @@ -777,16 +1097,26 @@ def entitlement_summary( health: Optional[Dict[str, InstrumentHealth]] = None, campaign: Optional[Campaign] = None, ) -> Dict[str, int]: - """Counts for the report. Every instrument in the workcell is in exactly one bucket.""" + """Counts for the report. Every instrument in the workcell is in exactly one bucket. + + Refuses a workcell with nothing in it. Zero in every bucket and a total of zero is + arithmetic over no instruments, and it reads exactly like a lab with nothing wrong -- + which is the shape of vacuous pass this package hunts, arriving as a summary instead of + as a check. + """ supplied = health or {} - keys: List[str] = list(workcell.present_keys()) - for key in workcell.federated: - if key not in keys: - keys.append(key) + _refuse_empty_campaign(campaign) + keys = _instrument_keys(workcell) + if not keys: + raise ValueError( + f"workcell '{workcell.name}' declares no present or federated instrument; counts of " + "zero over zero instruments read as a lab with nothing out of service, so declare " + "the instruments before summarizing them" + ) counts = {e.value: 0 for e in Entitlement} counts["total"] = len(keys) for key in keys: - h = supplied.get(key) or health_for(key) + h = _health_at(key, supplied) counts[h.entitlement(campaign).value] += 1 return counts @@ -892,9 +1222,14 @@ class Refusal: ), what_it_would_take=( "measured per-instrument reliability on this duty cycle AND a characterized " - "correlation structure between instruments, which no published autonomous-lab work " - "reports" + "correlation structure between the instruments, established on this bench or taken " + "from work that states the bench and duty cycle it was established on" ), + # The earlier wording claimed no published autonomous-lab work reports such a + # structure. That is a universal negative over a literature, carried in a string with + # no citation, inside a module whose argument is that an unsourced number becomes a + # specification. It is also unnecessary: naming what the figure would require is the + # refusal, and a claim about what a field has not shown is checkable nowhere here. ), ) diff --git a/autonomous_lab/feedback.py b/autonomous_lab/feedback.py index 696bdd5..c0f75b3 100644 --- a/autonomous_lab/feedback.py +++ b/autonomous_lab/feedback.py @@ -5,7 +5,8 @@ closed loop is three operations -- measure, compare, act -- and in a laboratory each one has its own specific way of being fake: - MEASURE the number arrives after the material it describes has already been consumed. + MEASURE the number arrives after the material it describes has already been consumed, + or the step named as the sensor never produces that number in the first place. COMPARE there is no envelope to compare against, so nothing defines what good is. ACT the correction is one the thing holding the loop may not make. @@ -33,21 +34,30 @@ than a control loop. Calling it closed-loop control hides the part that matters, which is that the current run is already spent. -The other two refusals are separate on purpose and are never collapsed into the first, +The other three refusals are separate on purpose and are never collapsed into the first, because each one is bought with different work. NO_ENVELOPE means nothing in this repo says what good is for that quantity, so there is no target to steer toward and the loop is underspecified rather than late; the fix is a threshold with a basis, which costs an -experiment. UNAVAILABLE_MEASUREMENT is resolved from the ledger -- the step the number comes +experiment. SENSOR_DOES_NOT_MEASURE_IT means the step the loop names as its sensor is not +where that number comes from -- qc owns the quantity-to-artifact fact and the protocol owns +which step produces that artifact, and between them they settle it without asking the loop's +author; the fix is a corrected declaration, or an assay this protocol does not currently +run. UNAVAILABLE_MEASUREMENT is resolved from the ledger -- the step the number really comes from is blocked, manual, or broken -- and the fix is a decode, a debug, or a person. And UNPERMITTED_CORRECTION means the loop declares a controller weaker than the correction requires, which is settled by a decision rather than by money. Reporting one number over -four different budgets is the failure this whole package exists to refuse. +five different budgets is the failure this whole package exists to refuse. The refusals are ordered so the headline is the one a reader must not be sent away from. A lab told "the measurement is unavailable" will go buy an instrument, and if the loop is also -structurally open that instrument buys nothing at all. So the structural refusal is reported -first and the fixable ones after it, and every refusal a loop earns is carried, not just the -first, so nobody discovers the next one only after paying for the last. +structurally open that instrument buys nothing at all -- and if the loop is sensing at a step +that never produced the number, repairing the instrument at that step buys nothing either. +So the structural refusal is reported first, the sensor refusal ahead of the instrument one, +and every refusal a loop earns is carried, not just the first, so nobody discovers the next +one only after paying for the last. `counts()` and `by_reason()` tally HEADLINES, which is +one refusal per loop; `blocker_counts()` tallies every refusal earned. The two answer +different questions and the docstrings say which is which, because a lab budgeting off the +headline tally reads a zero next to a refusal four loops earned. WHAT THIS MODULE WILL NOT SAY. It does not model gain, overshoot, damping, stability, or a settling time. Those are properties of a plant, and computing any of them needs a plant model @@ -64,24 +74,43 @@ count. It is the honest form of "how fast can this loop react": in plates, which the protocol knows, rather than in seconds, which nobody here has measured. -Two scoping notes, because a verdict read wider than it was computed is worse than no -verdict. CLOSES is a claim about the LOOP -- its order, its target, its measurement, and its +Three scoping notes, because a verdict read wider than it was computed is worse than no +verdict. CLOSES is a claim about the LOOP -- its order, its target, its sensor, and its authority. It is not a claim that the step being corrected runs at all; the ledger owns that question, and `corrections_nobody_can_apply` reports it beside the verdict rather than inside -it, the same way qc keeps a wrong-assay gate separate from an unsatisfiable one. And the -envelope a loop steers toward is resolved from the qc criterion that already owns the number, -never restated here: two copies of one cutoff eventually disagree, and a loop steering toward -the retired copy would look exactly like a loop steering toward the live one. +it, the same way qc keeps a wrong-assay gate separate from an unsatisfiable one. It is also +not a claim that the number is the RIGHT number for the sample. qc already owns that fact in +`Measurement.instrument_appropriate`, and a loop can drive a quantity into its band reliably +while the quantity is confidently about the wrong thing -- so that caveat is resolved from qc +and reported by `loops_on_an_inappropriate_measurement`, beside the verdict and named in the +CLOSES reason, never silently folded into either. And the envelope a loop steers toward is +resolved from the qc criterion that already owns the number, never restated here: two copies +of one cutoff eventually disagree, and a loop steering toward the retired copy would look +exactly like a loop steering toward the live one. Where no gate owns a bound, `for_quantity` +still resolves the UNITS off the qc measurement, because units are qc's fact whether or not +a criterion exists. """ from __future__ import annotations +import math from dataclasses import dataclass from enum import Enum from typing import Dict, List, Optional, Tuple from .model import Verdict -from .qc import GATES, MEASUREMENTS, PROTOCOL_GATES, Basis, Comparison, Criterion, Gate +from .qc import ( + GATES, + MEASUREMENTS, + PROTOCOL_GATES, + Basis, + Comparison, + Criterion, + Gate, + Measurement, + Readiness, +) +from .throughput import duration_for # The dynamic quantities this module refuses to emit, named so the refusal is explicit # rather than merely absent. Each one needs a plant model and measured response data -- @@ -106,11 +135,14 @@ class Controller(str, Enum): ratifier reads a proposal that has already framed the problem, and a plausible frame is what a model produces most reliably when it is wrong. - This names the same distinction the authority layer draws over decision classes, and it is - restated rather than imported on purpose -- a loop's controller is a property of the loop, - the ordering here is short enough to read in one screen, and this module has to be - checkable on its own. If the two are ever wired together, the authority layer owns the - requirement and this resolves from it rather than keeping a second copy. + Nothing else in this package decides correction authority, so `_MINIMUM_CONTROLLER` below + is the sole owner of these floors and there is no sibling to resolve them from. The + nearest neighbour is `provenance.Actor`, and it is worth naming precisely because it looks + like one: Actor records WHO performed a step that already happened -- machine, human, or + agent, where AGENT means a model proposed it and a gate or a person permitted it. It is a + label on a recorded event. It settles nothing about whether that actor was strong enough + to make the correction, which is the question this enum exists to answer, and it is asked + before the act rather than written down after it. """ MODEL = "model" # a model emits the correction and the correction is applied @@ -210,15 +242,23 @@ def minimum_controller(self) -> Controller: class Closable(str, Enum): """Whether a control loop can close, and if not, which of the three legs is fake. - The four refusals are listed in the order they are reported, and that order is the line + The five refusals are listed in the order they are reported, and that order is the line this enum draws. OPEN_LOOP_SENSOR_TOO_LATE is first because it is the only one no purchase, decode, or policy change repairs: the protocol has to be rewritten so the measurement happens before the thing it steers. Reporting a cheaper reason ahead of it sends a lab to buy an instrument for a loop that could never have closed. + + SENSOR_DOES_NOT_MEASURE_IT sits ahead of UNAVAILABLE_MEASUREMENT for the same reason one + step down. UNAVAILABLE_MEASUREMENT says the step the number comes from cannot be driven + here, and its fix is a decode, a debug, or a person AT THAT STEP. That advice is worthless + when the step named is not where the number comes from: repairing an instrument that never + produced the quantity produces it no harder. So the question "is this really the sensor" + is answered first, and the ledger is then read at the step the answer points to. """ OPEN_LOOP_SENSOR_TOO_LATE = "open_loop_sensor_too_late" # measured at or after the correction point NO_ENVELOPE = "no_envelope" # nothing declares what good is; there is nothing to steer toward + SENSOR_DOES_NOT_MEASURE_IT = "sensor_does_not_measure_it" # that step is not where this number comes from UNAVAILABLE_MEASUREMENT = "unavailable_measurement" # the measuring step does not produce a number here UNPERMITTED_CORRECTION = "unpermitted_correction" # the controller is weaker than the correction needs CLOSES = "closes" @@ -231,8 +271,11 @@ def closed(self) -> bool: def structural(self) -> bool: """True only for the refusal that money cannot move. - The other three name work a lab can buy, schedule, or decide. This one names an ordering - in the protocol, and the only thing that changes it is measuring somewhere else. + The other four name work a lab can buy, schedule, decide, or correct on the page. This + one names an ordering in the protocol, and the only thing that changes it is measuring + somewhere else. SENSOR_DOES_NOT_MEASURE_IT is deliberately not in here: a loop naming + the wrong sensor is usually a declaration a author fixes for free, and folding it in + with a refusal that needs the protocol rewritten would put the two behind one budget. """ return self is Closable.OPEN_LOOP_SENSOR_TOO_LATE @@ -241,17 +284,57 @@ def structural(self) -> bool: _REFUSAL_ORDER: Tuple[Closable, ...] = ( Closable.OPEN_LOOP_SENSOR_TOO_LATE, Closable.NO_ENVELOPE, + Closable.SENSOR_DOES_NOT_MEASURE_IT, Closable.UNAVAILABLE_MEASUREMENT, Closable.UNPERMITTED_CORRECTION, ) -# Which verdicts mean a number actually arrives from a step. Identical to the partition -# qc.Readiness draws over the same verdicts, and taken here off the ledger row the ledger -# already computed rather than by costing the step a second time. A step that is blocked, -# manual, written, or broken produces nothing to compare against, and a loop reading one is -# steering on a value nobody will ever hand it. -_MEASUREMENT_ARRIVES: Tuple[Verdict, ...] = (Verdict.AUTOMATED, Verdict.SUPERVISED) +def _readiness_of(verdict: Verdict) -> Readiness: + """Whether a number arrives from a step in this state, in qc's vocabulary. + + This is the partition `qc.readiness()` draws over the same verdicts, and the answer is + returned as a qc.Readiness rather than as a local tuple of verdicts so the two are one + vocabulary and not two. qc performs the mapping inline over a gate's inputs and exports no + function for it, so it cannot be imported today -- which is exactly why the RESULT TYPE is + imported and why a test costs a real gate through `qc.readiness()` and compares the answer + to this one. The day qc decides SUPERVISED is not evaluable, or a Verdict is added, that + test fails here rather than this module going on steering on the retired partition. + + Deliberately not reused to answer whether a machine can ACT at a step. That is a different + question about the same verdict, `_step_runs_today` answers it, and one constant answering + both is how the two start disagreeing with nothing to reveal which was meant. + """ + if verdict is Verdict.AUTOMATED: + return Readiness.READY + if verdict is Verdict.SUPERVISED: + return Readiness.SUPERVISED + return Readiness.UNSATISFIABLE + + +def _step_runs_today(verdict: Verdict) -> bool: + """Whether anything at all reaches a step in this state, machine or supervised operator. + + This is the partition `Ledger.reachable()` draws, asked here about the step a correction + lands on rather than about the step a number comes from. The ledger computes it inline and + exports no predicate, so a test pins this against `Ledger.reachable()` over a real ledger + rather than letting a second copy drift quietly. + """ + return verdict in (Verdict.AUTOMATED, Verdict.SUPERVISED) + + +def _correction_can_land(controller: "Controller", verdict: Verdict) -> bool: + """Can this controller actually apply a correction at a step in this state? + + Two facts and not one. The step has to be reachable at all, which is the ledger's + question, and where it is not, the only controller that changes the answer is a person -- + and only at a step whose whole description is that a person does it. MANUAL is that step. + BLOCKED, BROKEN and WRITTEN are not: an undecoded command set, a script that fails on the + instrument, and a script never run on it are not waiting for somebody to walk over. + """ + if _step_runs_today(verdict): + return True + return controller is Controller.HUMAN and verdict is Verdict.MANUAL @dataclass(frozen=True) @@ -263,10 +346,23 @@ class Envelope: on it reports itself in specification forever, corrects nothing, and looks exactly like a loop that is working. Absence of a bound is not a wide bound. + Neither is an infinite one, and neither is a NaN, so both are refused the same way and for + the same reason. `upper=inf` is not a permissive ceiling, it is the absence of a ceiling + spelled as a float, and it walks straight past a guard that only tests `is None`. NaN is + worse: every comparison against it is False, so it survives the inverted-bound check as + well and then `contains()` returns True for every value on both sides. A bound has to be + a real number for a band to be a band. + `basis` comes from the qc criterion this was resolved from and is carried rather than checked. An envelope resting on intuition is still an envelope and the loop can still close around it -- what it cannot do is pass unreported, so `unvalidated_envelopes()` lists it. + + The raw constructor stays open for a quantity qc does not declare -- residual_solvent_mass + is a real thing to bound and qc has never heard of it -- which is why `units` is a free + field here. Where qc DOES declare the quantity, build with `for_quantity` instead: the + units are qc's fact, and restating them by hand gives this lab a second copy that a reader + meets through `describe()` and through a closure's reason. """ quantity: str # the qc.Measurement key this bounds @@ -282,11 +378,51 @@ def __post_init__(self) -> None: f"envelope for '{self.quantity}' bounds nothing on either side; an unbounded band " "accepts every value, so a loop steering toward it never corrects anything" ) + for side, bound in (("lower", self.lower), ("upper", self.upper)): + if bound is not None and not math.isfinite(bound): + raise ValueError( + f"envelope for '{self.quantity}' has a {side} bound of {bound}, which bounds " + "nothing: an infinite side accepts every value above or below it and a NaN side " + "accepts every value at all, so a loop steering toward it never corrects anything" + ) if self.lower is not None and self.upper is not None and self.upper < self.lower: raise ValueError( f"envelope for '{self.quantity}' has upper {self.upper} below lower {self.lower}" ) + @classmethod + def for_quantity( + cls, + quantity: str, + lower: Optional[float] = None, + upper: Optional[float] = None, + basis: Basis = Basis.INTUITION, + note: str = "", + ) -> "Envelope": + """An envelope whose units are resolved from qc rather than typed in beside them. + + For the case `envelope_from_criterion` cannot serve: qc declares the quantity but no + gate declares a bound on it, so the number and its basis genuinely originate here. The + UNITS do not. qc owns them independently of whether any criterion exists, and a hand + written unit is a second copy that goes on printing after qc revises the measurement. + Refuses a quantity qc does not declare, so this cannot quietly become a second way to + invent one. + """ + measurement = MEASUREMENTS.get(quantity) + if measurement is None: + raise KeyError( + f"qc declares no measurement '{quantity}', so its units cannot be resolved; build " + "the envelope directly if the quantity is genuinely one qc does not own" + ) + return cls( + quantity=quantity, + units=measurement.units, + lower=lower, + upper=upper, + basis=basis, + note=note, + ) + @property def validated(self) -> bool: return self.basis.validated @@ -314,13 +450,27 @@ def envelope_from_criterion(gate: Gate, criterion_name: str) -> Envelope: day somebody revises the gate the loop would go on steering toward the retired band with nothing anywhere to reveal which of the two was live. Units are resolved the same way, off the declared measurement, so an envelope cannot be built for a quantity qc does not know. + + A criterion name used twice in one gate is refused rather than resolved to the first hit, + the same way `_index_of` refuses a step op used twice. The revision this function exists to + survive is precisely the one that produces a duplicate -- somebody adds the tightened + cutoff and leaves the permissive one in place -- so resolving silently would hand back the + retired band exactly when it matters most. """ - match: Optional[Criterion] = next((c for c in gate.criteria if c.name == criterion_name), None) - if match is None: + matches: List[Criterion] = [c for c in gate.criteria if c.name == criterion_name] + if not matches: raise KeyError( f"gate '{gate.name}' declares no criterion '{criterion_name}'; " f"it has: {[c.name for c in gate.criteria]}" ) + if len(matches) > 1: + raise ValueError( + f"gate '{gate.name}' declares '{criterion_name}' {len(matches)} times, with bounds " + f"{[c.bound for c in matches]}; a name used twice is not a criterion. Taking the first " + "would build the envelope out of whichever copy sits earlier in the tuple, and a loop " + "steering toward the retired cutoff looks exactly like one steering toward the live one" + ) + match: Criterion = matches[0] measurement = MEASUREMENTS.get(match.measurement) if measurement is None: raise KeyError( @@ -425,11 +575,32 @@ def closes(self) -> bool: def reason(self) -> str: if self.blockers: return self.blockers[0].detail - return ( + closes = ( f"'{self.loop.quantity}' is measured at '{self.loop.measured_at}', which is upstream of " f"'{self.loop.corrects}'; it steers toward {self.loop.steers_toward()}, the number " f"arrives, and {self.loop.applied_by.value} may make a {self.loop.correction.value}" ) + # The caveat travels with the verdict rather than beside it, because CLOSES is the one + # string a reader quotes and a wrong-assay loop closes perfectly well around the wrong + # number. Resolved from qc, so it disappears the day the appropriate assay lands. + wrong_assay = inappropriate_measurement(self.loop.quantity) + if wrong_assay is not None: + closes += ( + f". The loop closes and the number is the wrong one: qc reports " + f"'{wrong_assay.key}' is not an appropriate measurement here -- " + f"{wrong_assay.inappropriate_reason}. A closed loop drives this quantity into its " + "band reliably, and the band is about the wrong thing" + ) + return closes + + def on_an_inappropriate_measurement(self) -> Optional[Measurement]: + """The qc measurement this loop steers on, when qc calls it the wrong assay. + + A method rather than a field for the usual reason, and resolved from qc rather than + copied, so a loop stops being flagged the moment the measurement it reads is replaced + by one that is appropriate. + """ + return inappropriate_measurement(self.loop.quantity) def all_reasons(self) -> Tuple[str, ...]: return tuple(b.detail for b in self.blockers) @@ -443,9 +614,14 @@ class InFlight: """How much material is committed between a loop's sensor and its actuator. Counted in steps and reported in plates, never in seconds. Seconds would need step - durations, and `throughput` already refuses to return a plates-per-day figure because no - step in this repo has ever been timed. The same emptiness would sit under any reaction - time reported here, so this reports the thing the protocol actually knows. + durations, and neither reference protocol has a single step with a measured one -- + `throughput.estimate(...).why_not()` says 18 of 18 for the genomics flow. The package is + not entirely untimed, and the distinction matters because an absolute claim here is false: + `throughput.DURATIONS` carries one MEASURED entry, the Tecan's tray cycle, timed on real + hardware. It is the duration of a drawer opening and it belongs to no step in either + protocol, so it buys no reaction time for any loop. `meaning` counts the untimed steps of + the protocol in front of it rather than restating any of this, so the sentence a reader is + shown is computed and stops being true the day somebody times the flow. """ loop: Loop @@ -481,26 +657,149 @@ def _index_of(protocol, op: str, role: str) -> int: return hits[0] -def can_close(loop: Loop, protocol, ledger) -> Closure: - """Can this loop actually close against this protocol and this workcell? +def inappropriate_measurement(quantity: str) -> Optional[Measurement]: + """The qc measurement behind a quantity, when qc says it is the wrong assay for the sample. - Step ordering is resolved from the protocol and measurement availability from the ledger. - Neither is re-derived: the protocol owns the order the bench runs in, the ledger owns - whether a step produces anything, and a second computation of either would eventually - disagree with the first with nothing to say which was right. + Resolved from `qc.Measurement.instrument_appropriate` and never restated. This is the + quiet failure qc separates from an unsatisfiable gate, and it needs separating here for + the same reason: an unavailable measurement fails loudly, because no number arrives, while + a wrong-assay measurement arrives, compares cleanly, and steers the process confidently + toward a band computed about the wrong thing. Of the two, this is the one that spends the + flow cell. + """ + measurement = MEASUREMENTS.get(quantity) + if measurement is None or measurement.instrument_appropriate: + return None + return measurement + + +def _resolve_sensor(loop: Loop, protocol) -> Tuple[int, Optional[Blocker]]: + """Where this loop's number actually comes from, and the refusal if that is not where the + loop says. + + The declaration is not evidence. `measured_at` is a string an author types, and the shipped + loops being consistent with it says nothing about the next one -- a loop can name an HTTP + liveness probe as the sensor for a library concentration, and every downstream check will + then be costed against the probe's ledger row, which is AUTOMATED, which reads as "the + number arrives". That is the empty-measurement failure this package hunts, wearing a + control diagram: absent data yielding a verdict whose own prose asserts the data arrived. + + So where qc declares the quantity, the sensor is resolved the way `qc.readiness()` resolves + it and not from the declaration: the measurement names the artifact it comes from, the + protocol names the step that produces that artifact, and that step is the sensor. Every + later check is costed against THAT row. Where the two disagree the loop is refused and the + refusal names both, because one of them is wrong and this module cannot tell which. + + Where qc does NOT declare the quantity there is no artifact to resolve through, and + residual_solvent_mass is a real quantity in exactly that position. The declaration is then + all there is, and the one thing still checkable is whether the named step yields anything + at all: a step whose `produces` is empty hands back no artifact, so it hands back no + number, and a loop reading one is reading a value the protocol never creates. That is a + necessary condition and not a sufficient one, and the docstring says so rather than letting + a weaker check be read as the stronger one. + """ + declared_index = _index_of(protocol, loop.measured_at, "measured_at") + measurement = MEASUREMENTS.get(loop.quantity) - Every check runs. The function does not return early on the first refusal, because a loop - broken three ways that reported one way would be fixed one way and still be broken. + if measurement is None: + if not protocol.steps[declared_index].produces: + return declared_index, Blocker( + Closable.SENSOR_DOES_NOT_MEASURE_IT, + f"'{loop.quantity}' is not a quantity qc declares, so nothing resolves where it comes " + f"from, and the step this loop names as its sensor -- '{loop.measured_at}' -- produces " + "no artifact at all. Nothing in this protocol creates that number, so the comparison " + "would run against a value nobody hands it. Declare the quantity in qc, or name a " + "step that actually produces something", + ) + return declared_index, None + + producers = [ + i for i, step in enumerate(protocol.steps) if measurement.produced_by in step.produces + ] + if not producers: + return declared_index, Blocker( + Closable.SENSOR_DOES_NOT_MEASURE_IT, + f"qc says '{loop.quantity}' comes from artifact '{measurement.produced_by}', which no " + f"step in protocol '{protocol.name}' produces. The loop names '{loop.measured_at}' as " + "its sensor, but this protocol never creates that number anywhere, so no instrument " + "repair at that step or any other makes it arrive", + ) + if len(producers) > 1: + raise ValueError( + f"artifact '{measurement.produced_by}' is produced at {len(producers)} positions in " + f"protocol '{protocol.name}' ({[protocol.steps[i].op for i in producers]}), so " + f"'{loop.quantity}' has no single source and cannot be resolved to a sensor. Picking " + "one would cost the loop against whichever row happened to come first" + ) + resolved_index = producers[0] + if resolved_index != declared_index: + return resolved_index, Blocker( + Closable.SENSOR_DOES_NOT_MEASURE_IT, + f"this loop declares '{loop.quantity}' is measured at '{loop.measured_at}', but qc says " + f"it comes from artifact '{measurement.produced_by}', which this protocol produces at " + f"'{protocol.steps[resolved_index].op}' (step {resolved_index + 1}). " + f"'{loop.measured_at}' produces {list(protocol.steps[declared_index].produces) or 'nothing'}" + ", so it is not where this number is taken. Costing the loop against the declared step " + "would read that step's verdict as though it were this measurement's, which is how a " + "loop over a number nobody produces reports that the number arrives", + ) + return declared_index, None + + +def _refuse_mismatched_ledger(protocol, ledger) -> None: + """Refuse a ledger that does not line up, row for row, with the protocol being costed. + + Every row lookup in this module is POSITIONAL -- the step at index i is costed by the row + at index i -- so the two have to be the same list of steps and not merely two things with + the same name. Comparing names alone leaves the whole check open: two protocols may share + a name and share no steps, and the module then reads an unrelated step's verdict as though + it were the sensor's. A blocked sensor borrowed from a protocol whose first step happens to + be a socket probe reads as AUTOMATED, and the loop is reported closed on the strength of an + instrument it never touches. A ledger shorter than the protocol truncates silently and one + longer reads past the end, so the length is part of the identity too. """ if ledger.protocol.name != protocol.name: raise ValueError( f"ledger was built for protocol '{ledger.protocol.name}' but the loop is being costed " f"against '{protocol.name}'; a verdict mixed from two protocols describes neither" ) + ledger_ops = tuple(row.step.op for row in ledger.rows) + protocol_ops = tuple(step.op for step in protocol.steps) + if ledger_ops != protocol_ops: + raise ValueError( + f"the ledger named '{ledger.protocol.name}' was built over a different step list than " + f"the protocol of that name being costed here: {list(ledger_ops)} against " + f"{list(protocol_ops)}. Rows are matched to steps by position, so costing across these " + "two would read one step's verdict as another step's, which is the mixed verdict the " + "name check exists to refuse" + ) + - measured_index = _index_of(protocol, loop.measured_at, "measured_at") +def can_close(loop: Loop, protocol, ledger) -> Closure: + """Can this loop actually close against this protocol and this workcell? + + Step ordering is resolved from the protocol, which step a quantity comes from is resolved + from qc, and whether that step runs is resolved from the ledger. None of the three is + re-derived and none of the three is taken from the loop's author: the protocol owns the + order the bench runs in, qc owns which artifact carries which number, the ledger owns + whether a step produces anything, and a second computation of any of them would eventually + disagree with the first with nothing to say which was right. + + That is why `measured_index` is the RESOLVED sensor rather than the declared one. Every + check downstream of it -- can the number reach the material, does the number arrive -- is + a question about the step the number actually comes from, and asking them at a step an + author merely named is how a loop over an unproduced quantity reports that it closes. + + Every check runs. The function does not return early on the first refusal, because a loop + broken three ways that reported one way would be fixed one way and still be broken. + """ + _refuse_mismatched_ledger(protocol, ledger) + + measured_index, sensor_blocker = _resolve_sensor(loop, protocol) corrects_index = _index_of(protocol, loop.corrects, "corrects") blockers: List[Blocker] = [] + if sensor_blocker is not None: + blockers.append(sensor_blocker) # -- 1. can the measurement reach the material it is supposed to steer? ------ # The one refusal no purchase moves, so it is checked and reported first. @@ -551,14 +850,17 @@ def can_close(loop: Loop, protocol, ledger) -> Closure: ) # -- 3. does the number actually arrive? ------------------------------------- + # Read at the RESOLVED sensor, so a mis-declared loop is costed against the step the + # number really comes from rather than against the one it named. measuring_row = ledger.rows[measured_index] - if measuring_row.verdict not in _MEASUREMENT_ARRIVES: + if not _readiness_of(measuring_row.verdict).evaluable: blockers.append( Blocker( Closable.UNAVAILABLE_MEASUREMENT, - f"'{loop.quantity}' comes from '{loop.measured_at}', which this workcell costs as " - f"{measuring_row.verdict.value}: {measuring_row.reason}. A loop over a number nobody " - "produces is an open loop that reports no error, because the comparison never runs", + f"'{loop.quantity}' comes from '{measuring_row.step.op}', which this workcell costs " + f"as {measuring_row.verdict.value}: {measuring_row.reason}. A loop over a number " + "nobody produces is an open loop that reports no error, because the comparison never " + "runs", ) ) @@ -600,8 +902,13 @@ def in_flight_exposure(loop: Loop, protocol) -> InFlight: Refused for a loop whose sensor is not upstream of its actuator. There is no in-flight count for a correction that never reaches its own material, and returning zero would read as the best possible case rather than as the absence of a loop. + + The sensor is resolved the same way `can_close` resolves it, so the two agree about which + step the number comes from. A loop that names the wrong sensor gets its exposure counted + from the step qc says the number is actually taken at, and `can_close` is where that + disagreement is reported. """ - measured_index = _index_of(protocol, loop.measured_at, "measured_at") + measured_index, _ = _resolve_sensor(loop, protocol) corrects_index = _index_of(protocol, loop.corrects, "corrects") if measured_index >= corrects_index: raise ValueError( @@ -611,21 +918,32 @@ def in_flight_exposure(loop: Loop, protocol) -> InFlight: ) intervening = tuple(step.op for step in protocol.steps[measured_index + 1 : corrects_index]) count = len(intervening) + # Counted off throughput rather than asserted. "Nothing here has ever been timed" is a + # claim about a sibling's data, it is false about this package as a whole -- one duration + # in DURATIONS is MEASURED -- and a claim of that shape printed into a string a reader is + # shown is exactly the kind this repo refuses. So the sentence carries the count for the + # protocol in front of it, and stops saying this the day somebody times the flow. + untimed = tuple(step.op for step in protocol.steps if not duration_for(step.op).known) + timing = ( + f"{len(untimed)} of {len(protocol.steps)} steps in this protocol have never been timed, " + "so nothing here converts plates into seconds; throughput refuses the same conversion " + "over the same steps" + if untimed + else "every step in this protocol has a measured duration; ask throughput for seconds" + ) if count == 0: meaning = ( f"the correction lands at '{loop.corrects}', the step immediately after the " f"measurement. Nothing is committed in between, so every plate this loop measures is a " - "plate it can still steer. This is the tightest loop the protocol admits" + f"plate it can still steer. This is the tightest loop the protocol admits. {timing}" ) else: meaning = ( - f"{count} step position(s) sit between '{loop.measured_at}' and '{loop.corrects}': " - f"{', '.join(intervening)}. On a line running full that is {count} plate(s) already " - f"past the sensor and not yet at the correction point; each was admitted before this " - f"measurement existed and reaches '{loop.corrects}' under the setting the correction " - "is about to change. No controller helps them. Nothing here converts that into " - "seconds: no step in this package has ever been timed, and throughput refuses the same " - "conversion for the same reason" + f"{count} step position(s) sit between '{protocol.steps[measured_index].op}' and " + f"'{loop.corrects}': {', '.join(intervening)}. On a line running full that is {count} " + f"plate(s) already past the sensor and not yet at the correction point; each was " + f"admitted before this measurement existed and reaches '{loop.corrects}' under the " + f"setting the correction is about to change. No controller helps them. {timing}" ) return InFlight(loop=loop, steps_between=count, intervening=intervening, meaning=meaning) @@ -647,6 +965,26 @@ def in_flight_exposure(loop: Loop, protocol) -> InFlight: LOOPS: Dict[str, Loop] = {} + +def _register(registry: Dict[str, Loop], loop: Loop) -> None: + """Add a loop to a registry, refusing a name that is already taken. + + A bare assignment into the dict makes a duplicate name a silent replacement, and the + replacement deletes a finding without a word: register a benign loop under the name of + this repo's flagship open loop and `structurally_open()` quietly loses a row while + `PROTOCOL_LOOPS` goes on listing the name once, so nothing anywhere reveals the swap. + `_index_of` refuses a duplicated step op loudly for exactly this reason; the registry that + feeds it has to do the same. + """ + if loop.name in registry: + raise ValueError( + f"a loop named '{loop.name}' is already registered, measuring '{registry[loop.name].quantity}' " + f"at '{registry[loop.name].measured_at}'. A second registration replaces the first " + "silently and deletes whatever the first one reported; rename one of them" + ) + registry[loop.name] = loop + + for _loop in ( Loop( name="pooled_quant_steers_pooling", @@ -715,15 +1053,16 @@ def in_flight_exposure(loop: Loop, protocol) -> InFlight: corrects="start_run", correction=Correction.REPEAT, applied_by=Controller.MODEL_PROPOSES, - envelope=Envelope( + envelope=Envelope.for_quantity( quantity="run_yield_pf", - units="Gbp", lower=1.0, basis=Basis.INTUITION, note=( - "a floor on usefulness rather than a specification. Deliberately not resolved from a " - "qc criterion, because no gate in this repo reads run yield -- and an envelope with " - "no gate behind it is exactly the kind that never gets revised" + "a floor on usefulness rather than a specification. The BOUND is deliberately not " + "resolved from a qc criterion, because no gate in this repo reads run yield -- and an " + "envelope with no gate behind it is exactly the kind that never gets revised. The " + "units are qc's regardless: qc declares run_yield_pf whether or not anything gates " + "on it, and typing 'Gbp' here would be a second copy that outlives the first" ), ), note=( @@ -743,14 +1082,17 @@ def in_flight_exposure(loop: Loop, protocol) -> InFlight: envelope=None, note=( "the chemistry loop, and the one that shows what a missing envelope actually looks " - "like. Its order is fine. What is absent is a target: qc declares no gate at all for " - "this protocol, because nothing reads the chromatogram back, so there is no number in " - "this repo that says what a good dry-down is. A loop here would run, act, and have no " - "way to be wrong" + "like. Its ORDER is fine and that is the only leg of it that is. What is absent is a " + "target: qc declares no gate at all for this protocol, because nothing reads the " + "chromatogram back, so there is no number in this repo that says what a good dry-down " + "is. Its sensor is absent too, which the module now says out loud -- get_status polls " + "the V-10 until the method finishes and produces no artifact, so the residual mass it " + "steers on is not a number this protocol yields either. A loop here would run, act, " + "and have no way to be wrong" ), ), ): - LOOPS[_loop.name] = _loop + _register(LOOPS, _loop) # Keyed by protocol so a protocol with no declared loops is visibly unlooped rather than @@ -767,6 +1109,22 @@ def in_flight_exposure(loop: Loop, protocol) -> InFlight: } +# A name listed twice, or listed and never registered, is the same silent deletion the +# registry guard refuses one level down: the report would cost one loop twice, or fail on a +# lookup at report time rather than here where the mistake is. +for _protocol_name, _names in PROTOCOL_LOOPS.items(): + if len(set(_names)) != len(_names): + raise ValueError( + f"protocol '{_protocol_name}' lists a loop name more than once: {list(_names)}; the " + "same loop costed twice reads as two findings" + ) + for _name in _names: + if _name not in LOOPS: + raise ValueError( + f"protocol '{_protocol_name}' lists loop '{_name}', which is not registered" + ) + + @dataclass class FeedbackReport: """Every declared loop on a protocol, costed against a real workcell.""" @@ -790,47 +1148,130 @@ def structurally_open(self) -> List[Closure]: return [r for r in self.rows if r.verdict.structural] def by_reason(self) -> Dict[str, List[Closure]]: + """Rows grouped by their HEADLINE refusal, which is one refusal per loop. + + Reads the same way `counts()` does and carries the same caveat: a loop refused three + ways appears once, under the first. `blocker_counts()` is the view over everything a + loop earned. + """ out: Dict[str, List[Closure]] = {} for row in self.rows: out.setdefault(row.verdict.value, []).append(row) return out def unvalidated_envelopes(self) -> List[Closure]: - """Loops steering toward a band this lab has not earned with its own data. + """Loops not steering toward a band this lab has earned with its own data. A caveat and not a refusal, deliberately. A loop around a vendor or intuition threshold genuinely closes -- it drives the quantity into that band, reliably, whether or not the - band is the right one. Making it a fifth refusal would say the fix is the same as for a - loop with no target at all, and it is not: this one needs a titration, not a threshold. + band is the right one. Making it a refusal would say the fix is the same as for a loop + with no target at all, and it is not: this one needs a titration, not a threshold. + + A loop with NO envelope is in this list too, and the reason is the failure mode this + package hunts. Filtering on `envelope is not None` meant a loop that declared nothing + was not merely unflagged, it was counted among the clean ones -- so deleting the field + the metric is about improved the metric, and a lab looked better taught about its own + thresholds for having said less. No basis at all is strictly worse than a vendor basis, + never better. `envelopes_by_state()` keeps the two apart for a reader who needs them + apart. """ return [ - r for r in self.rows if r.loop.envelope is not None and not r.loop.envelope.validated + r for r in self.rows if r.loop.envelope is None or not r.loop.envelope.validated ] + def envelopes_by_state(self) -> Dict[str, List[Closure]]: + """Every row partitioned three ways: validated band, unvalidated band, no band at all. + + Three states and not two, so an absent envelope can never be summarised as a validated + one by a caller counting whatever is left over. Each needs different work: nothing, a + titration, and an experiment plus a stated basis. + """ + out: Dict[str, List[Closure]] = {"validated": [], "unvalidated": [], "absent": []} + for row in self.rows: + if row.loop.envelope is None: + out["absent"].append(row) + elif row.loop.envelope.validated: + out["validated"].append(row) + else: + out["unvalidated"].append(row) + return out + + def loops_on_an_inappropriate_measurement(self) -> List[Closure]: + """Loops steering on a number qc says is the wrong assay for this sample. + + The analogue of `qc.GateReport.inappropriate`, and it is here because a reader of this + module alone would otherwise see CLOSES and nothing else. The two failures are opposites + in how they present: an unavailable measurement fails loudly, because no number arrives, + while this one arrives, compares cleanly, and closes the loop around a quantity that is + confidently about something else. Beside the verdict rather than inside it, because the + loop does close -- it drives the number into the band every time -- and the fix is a + different assay rather than a different loop. + """ + return [r for r in self.rows if r.on_an_inappropriate_measurement() is not None] + def corrections_nobody_can_apply(self) -> List[Closure]: - """Loops whose correction lands on a step no code path reaches, with no human declared. + """Loops whose correction lands on a step nothing declared can actually perform. Kept out of the verdict on purpose, the same way qc keeps a wrong-assay gate out of readiness. Whether a step can be performed is the ledger's question and it is already answered there; re-deciding it inside a closure verdict would put an execution problem - and a control problem behind one word. A human-applied correction at a manual step is a - perfectly good closed loop, just a slow one, so only a declared machine controller - counts here. + and a control problem behind one word. + + The carve-out for a human controller is exactly as wide as the argument for it and no + wider. That argument is about MANUAL: a manual step is one a person does at the bench, + so a person applying the correction there is a real closed loop, just a slow one. It + says nothing about BLOCKED, BROKEN or WRITTEN, which are a step whose command set is + undecoded, a script that fails on the instrument, and a script that has never been run + on it. No bench scientist ratifies any of those away by being named in a declaration. + Skipping every HUMAN row made `applied_by` an author-settable field that emptied this + whole report -- one word in a declaration, and five stranded corrections became none -- + which is the override this module is supposed to make impossible. """ return [ - r - for r in self.rows - if r.loop.applied_by is not Controller.HUMAN - and r.corrects_verdict not in _MEASUREMENT_ARRIVES + r for r in self.rows if not _correction_can_land(r.loop.applied_by, r.corrects_verdict) ] def counts(self) -> Dict[str, int]: + """HEADLINE tally: one refusal per loop, the one it is reported under. + + Named here rather than left to be inferred, because this is the number a lab budgets + from and it is not the number of problems. A loop refused for its sensor AND its + authority is counted once, under the sensor, and this tally shows a zero next to the + authority problem it has. `blocker_counts()` is the other view, and the module's promise + that every refusal is carried is kept there. + """ out = {reason.value: 0 for reason in Closable} for row in self.rows: out[row.verdict.value] += 1 out["total"] = len(self.rows) return out + def blocker_counts(self) -> Dict[str, int]: + """Every refusal every loop earned, tallied in reporting order. + + The tally that keeps this module's own promise. `counts()` reports headlines, and on the + genomics protocol it says zero unpermitted corrections while a loop is sitting there + with one -- so a lab reading it concludes there is no authority problem and budgets only + the decode, which is the sequential rediscovery this module exists to prevent. Sums to + more than the row count whenever a loop is broken more than one way, and that is the + information: the gap between this and `counts()` is how much is hiding behind headlines. + """ + out = {reason.value: 0 for reason in _REFUSAL_ORDER} + for row in self.rows: + for blocker in row.blockers: + out[blocker.reason.value] += 1 + return out + + def unlooped(self) -> bool: + """True when nobody has declared a loop for this protocol at all. + + An empty report and a spotless report are otherwise the same object, and of the two the + empty one is worse: a protocol with no declared loop is open-loop by construction, + however automated it is, and every accessor here reports perfection over zero rows. + Resolved from PROTOCOL_LOOPS, which is the thing that would be missing. + """ + return not PROTOCOL_LOOPS.get(self.protocol, ()) + def closes_any(self) -> bool: """True only if at least one declared loop actually closes. @@ -841,7 +1282,16 @@ def closes_any(self) -> bool: def feedback_report(protocol, ledger) -> FeedbackReport: - """Cost every loop declared for a protocol against the ledger built for it.""" + """Cost every loop declared for a protocol against the ledger built for it. + + The ledger is checked against the protocol HERE, before the loop list is consulted, and + not only inside `can_close`. A protocol nobody has declared a loop for produces no rows, + so `can_close` is never reached, so the mismatch guard never runs -- and a caller who + handed in a ledger built for something else got a spotless report out of a check that + would have refused every row it was given. A guard reachable only when there is something + to guard is not a guard. + """ + _refuse_mismatched_ledger(protocol, ledger) names = PROTOCOL_LOOPS.get(protocol.name, ()) return FeedbackReport( protocol=protocol.name, @@ -881,6 +1331,7 @@ def ungated_protocols() -> Tuple[str, ...]: "can_close", "envelope_from_criterion", "feedback_report", + "inappropriate_measurement", "in_flight_exposure", "loops_for", "ungated_protocols", diff --git a/autonomous_lab/teaching.py b/autonomous_lab/teaching.py index 5e8392c..5c92355 100644 --- a/autonomous_lab/teaching.py +++ b/autonomous_lab/teaching.py @@ -24,6 +24,26 @@ mean plus k standard deviations is a fact plus a k somebody chose, and k is exactly the kind of number that reaches a report with no author attached to it. +THREE ENTRIES IS NOT THREE MEASUREMENTS. A minimum that counts rows is not a minimum, +because one number pasted three times satisfies it while carrying the evidence of one +number. So a record has to be a measurement before it is counted: it names where it came +from, its value is a finite number, and it is not byte-identical to another record already +in the same envelope. Two identical records are one measurement recorded twice or two runs +nobody troubled to distinguish, and neither of those is two measurements. The same reasoning +refuses a range of width zero over enough runs: independent measurements of a continuous +quantity do not agree to full float precision, so that range is a transcribed number rather +than a demonstrated one, and reporting it would state the tightest envelope in the module on +the weakest evidence in it. + +AN ENVELOPE IS A RANGE OVER ONE EXPERIMENT. `conditions` is the field that says which +experiment, and pooling two of them is the failure this module exists to prevent rather than +a convenience. A nanogram demonstration and a picogram demonstration in one envelope produce +a range wider than either, and wide is not the safe direction: a narrow envelope asks for +more demonstrations, a wide one hides a failure. `Envelope` therefore refuses mixed +conditions the way it refuses mixed metrics, and `_build_envelopes` keys by operation AND +conditions, so an operation demonstrated two ways has two envelopes and no single answer to +"the envelope for this operation" -- which is the honest answer rather than a gap. + ONE GOOD MACHINE RUN IS NOT PARITY, which is why `Attainment` has four values rather than three. A machine that ran the operation once, well, has produced a number that sits inside the expert's range; reporting that as MEETS claims the machine performs like the person, on @@ -35,18 +55,31 @@ rather than its mean, because a mean lets one excellent run pay for a bad one, and at the bench the bad plate is the one that costs the sample. +BOTH SIDES OF THE COMPARISON GET THE SAME CHECKS. `Envelope` refusing to pool two metrics is +worth nothing while `attainment` will place a percent against a fraction, and the machine +side is where the flattering number enters, because it is the side a script writes rather +than the side a person writes down after measuring. So an observation whose operation, +metric, or units differ from the envelope's is refused rather than compared, and the prose +reports the observation's own units beside the observation's own value, so a mismatch cannot +be papered over in a sentence. + BENCHMARK OR ENVELOPE. `intelligence.Benchmark` and `Envelope` look like one object and are two kinds of evidence. A benchmark is a target somebody ASSERTED -- a CV below 5 percent, a recovery inside specification. An envelope is a target somebody DEMONSTRATED -- the range a competent person actually produced, on this material, on this bench. As evidence that a -number is ACHIEVABLE the envelope is strictly stronger, and the reason is arithmetic rather -than rhetoric: an assertion has no n. It cannot be wrong about the spread because it never -states one, and a claim that cannot be wrong is not evidence. As a statement of what the -science NEEDS the benchmark is the stronger one, and no quantity of demonstrations replaces -it -- an expert can demonstrate 40 percent recovery repeatably and 40 percent can still be -too little for the assay. They answer different questions and a lab needs both. What it must -not do is let one stand in for the other, so `transfer_report` carries them side by side and -resolves the benchmark from `intelligence` rather than restating its target here. +number is ACHIEVABLE the envelope is usually the stronger, and the reason is arithmetic +rather than rhetoric: an UNMEASURED assertion has no n. It cannot be wrong about the spread +because it never states one, and a claim that cannot be wrong is not evidence. The qualifier +carries weight, because `intelligence` records a STATUS on every benchmark and two of them +are met with a counted series behind them. So `asserted_only` resolves `BenchmarkStatus` +rather than the presence of a Benchmark object, and a met benchmark with no envelope gets +its own row state instead of being filed as a target nobody has hit. As a statement of what +the science NEEDS the benchmark is the stronger one, and no quantity of demonstrations +replaces it -- an expert can demonstrate 40 percent recovery repeatably and 40 percent can +still be too little for the assay. They answer different questions and a lab needs both. +What it must not do is let one stand in for the other, so `transfer_report` carries them +side by side and resolves the benchmark from `intelligence` and the enforced threshold from +`qc` rather than restating either target here. THE BACKLOG NOBODY WRITES DOWN. `taught()` answers whether an operation has an expert envelope at all, and in this repo the answer is no for every operation, because there is not @@ -57,6 +90,14 @@ automation. `demonstration_queue` ranks it the way `ledger.unlocks` ranks decoding work: by what one sitting would free, rather than by what is convenient to schedule. +A backlog that quietly drops what it cannot schedule is the flattering version of that +queue, so an operation with no `Transferable` is not skipped. A missing spec and a +deliberate exemption look identical from inside a loop and are opposite claims: the first is +the least taught state in the protocol, the second is a step whose outcome the instrument +decides. `EXEMPT` makes the second explicit and readable, and anything in neither table +lands in `DemonstrationQueue.unspecified`, where "nobody has said what to measure" reads as +work rather than as absence. + The machine side is empty here for a reason worth stating. This lab has timed exactly one mechanism, and that measurement lives in `throughput.DURATIONS`, which records a value and a worst case rather than the series behind them. A summary cannot supply an n, so it cannot be @@ -67,11 +108,14 @@ from __future__ import annotations +import math from dataclasses import dataclass from enum import Enum from typing import Dict, List, Optional, Sequence, Tuple from .intelligence import BENCHMARKS_BY_OP, Benchmark +from .qc import GATES as QC_GATES +from .qc import MEASUREMENTS, Basis, Criterion # How many demonstrations of one operation before a tolerance may be stated at all. Three is # a floor set by convention rather than by a power calculation, and the module reports the @@ -99,7 +143,13 @@ class Goal(str, Enum): @property def outside_is_always_worse(self) -> bool: - """True when leaving the demonstrated range in either direction is a failure.""" + """True when leaving the demonstrated range in either direction is a failure. + + `attainment` branches on this rather than restating the two-sided rule as an unnamed + `else`. A fourth Goal added later then reaches an explicit refusal instead of silently + inheriting WINDOW semantics from a fall-through while this property reports False for + it -- two computations of one fact, disagreeing with nothing to reveal it. + """ return self is Goal.WINDOW @@ -128,6 +178,41 @@ def demonstrated_parity(self) -> bool: return self is Attainment.MEETS +def _measured(kind: str, record) -> None: + """The checks a record must survive to be a measurement rather than only a row. + + Shared by both record types because they are deliberately separate evidence about + separate agents, and the reasons a value fails to be a measurement are not separate at + all. A check written on one side and not the other is a check the flattering number walks + around. + + `by` is normalized rather than merely stored, because it is the only mechanism separating + "three runs by one person" from "three runs by three people". A trailing space would + otherwise upgrade one demonstrator's habits into the spread of the operation, and the + caveat that fires on a single demonstrator is the one that must travel with every parity + claim made against the envelope. + """ + handle = record.by.strip().casefold() + if not handle: + raise ValueError( + f"a {kind} of '{record.operation}' records no handle for who or what produced it; an " + "unattributed measurement cannot be counted toward a spread across demonstrators" + ) + object.__setattr__(record, "by", handle) + if not record.evidence.strip(): + raise ValueError( + f"a {kind} of '{record.operation}' by {handle} records no evidence; a value with no " + "run id, notebook page, or dataset behind it is a number nobody can go back to, and " + "every function in this module would compute on it and report a result" + ) + if not math.isfinite(record.value): + raise ValueError( + f"a {kind} of '{record.operation}' by {handle} has value {record.value}; a run whose " + "outcome could not be measured is not an observation of the operation, and counting " + "it would pad the n the minimum exists to hold" + ) + + @dataclass(frozen=True) class Demonstration: """One expert, performing one operation once, with the outcome measured. @@ -142,6 +227,11 @@ class Demonstration: another run -- the material, the input amount, the labware. A demonstration at nanogram input is not evidence about picogram input, and the whole difficulty of low-input work is that the two look like the same operation. + + `evidence` is required rather than decorative. `qc.Criterion` carries a basis, + `intelligence.Benchmark` carries evidence, `throughput.Duration` carries a basis; a value + here with no source would be the only number in the package nobody could go back to, and + it would still reach a MEETS. """ operation: str @@ -150,7 +240,10 @@ class Demonstration: value: float by: str # demonstrator handle, not a name conditions: str - evidence: str = "" + evidence: str # run id, notebook page, dataset path: where this number came from + + def __post_init__(self) -> None: + _measured("demonstration", self) @dataclass(frozen=True) @@ -161,6 +254,8 @@ class MachineObservation: are different evidence about different agents, and one container for both invites the arithmetic that pools them -- which would compute a range over expert and machine runs together and report the machine as being inside it. + + "The same metric as the expert" is enforced by `attainment`, not left to this sentence. """ operation: str @@ -169,7 +264,28 @@ class MachineObservation: value: float by: str # the run card, script, or instrument that produced it conditions: str - evidence: str = "" + evidence: str # run id, log, dataset path: where this number came from + + def __post_init__(self) -> None: + _measured("machine observation", self) + + +def _repeats(records: Sequence) -> Tuple: + """Records byte-identical to an earlier one in the same sequence. + + Equality is every field, evidence included, so two real runs differ by their source even + when they land on the same value. What this catches is one row entered twice, which is + either a data-entry error or a claim that one run happened three times, and neither is the + independent n the minimum is counting. + """ + seen: List = [] + repeats: List = [] + for r in records: + if r in seen: + repeats.append(r) + else: + seen.append(r) + return tuple(repeats) @dataclass(frozen=True) @@ -182,8 +298,16 @@ class Envelope: class reports is a method over `demonstrations`, and an envelope with too few of them returns None and says why rather than returning a plausible interval. - `goal` and the metric come from the operation's `Transferable` spec, not from the - demonstrations, so a demonstration cannot redefine what better means by being recorded. + `goal`, `metric`, and `units` come from the operation's `Transferable` spec, and all three + are checked against it HERE rather than only inside `_build_envelopes`. `goal` is the + single field that decides whether leaving the range is a failure: relabelling a WINDOW + operation as HIGHER turns this module's own documented over-delivery failure into + demonstrated parity, and a class in `__all__` that accepts that label is an + author-settable silencer whatever its docstring promises. + + Every demonstration must also share one `conditions` value. An envelope is a range over + one experiment; a range over two is wider than either, and wide is the direction that + hides a failure. """ operation: str @@ -193,6 +317,19 @@ class reports is a method over `demonstrations`, and an envelope with too few of demonstrations: Tuple[Demonstration, ...] = () def __post_init__(self) -> None: + spec = TRANSFERABLE_BY_OP.get(self.operation) + if spec is None: + raise ValueError( + f"no Transferable declares what a demonstration of '{self.operation}' measures; an " + "envelope over a metric nobody specified cannot be compared to anything" + ) + if self.metric != spec.metric or self.units != spec.units or self.goal is not spec.goal: + raise ValueError( + f"the spec for '{self.operation}' measures {spec.metric} in {spec.units} where " + f"{spec.goal.value} is better; this envelope declares {self.metric} in {self.units} " + f"where {self.goal.value} is better. Which direction is better is not a property of " + "the demonstrations, and an envelope that redefines it states a verdict, not a range" + ) for d in self.demonstrations: if d.operation != self.operation: raise ValueError( @@ -204,6 +341,22 @@ def __post_init__(self) -> None: f"demonstration by {d.by} records {d.metric} in {d.units}; two metrics pooled into " "one range is a number about nothing" ) + conditions = self.conditions() + if len(conditions) > 1: + raise ValueError( + f"envelope for '{self.operation}' pools demonstrations from {len(conditions)} " + f"experiments ({'; '.join(conditions)}); an envelope is a range over one set of " + "conditions, and pooling two widens it past the failure it was there to catch" + ) + repeats = _repeats(self.demonstrations) + if repeats: + first = repeats[0] + raise ValueError( + f"envelope for '{self.operation}' holds {len(repeats)} demonstration(s) identical to " + f"an earlier entry (by {first.by}, value {first.value}, evidence '{first.evidence}'); " + f"one measurement recorded twice is not two measurements, and {MIN_DEMONSTRATIONS} " + "of them is not a tolerance" + ) # -- derived, and therefore methods ---------------------------------------- @@ -235,17 +388,26 @@ def center(self) -> Optional[float]: return sum(d.value for d in self.demonstrations) / len(self.demonstrations) def tolerance(self) -> Optional[Tuple[float, float]]: - """The demonstrated range, or None when there are too few demonstrations to state one. + """The demonstrated range, or None where the demonstrations do not support one. min and max of the observations, with no widening and no distribution assumed. The interval is then a record of what happened rather than a model of what might, and it is wrong only in the direction of being too narrow -- which is the safe direction, because - a narrow envelope asks for more demonstrations and a wide one hides a failure. + a narrow envelope asks for more demonstrations and a wide one hides a failure. That + argument holds only because the constructor refuses to pool two sets of conditions: + pooling is precisely how min and max get wider than either experiment. + + A range of exactly zero over enough demonstrations is refused rather than reported. It + would be the tightest envelope in the module resting on the least evidence in it, and on + a continuous quantity it is a copied number rather than a reproduced one. """ if len(self.demonstrations) < MIN_DEMONSTRATIONS: return None values = [d.value for d in self.demonstrations] - return (min(values), max(values)) + low, high = min(values), max(values) + if low == high: + return None + return (low, high) def spread(self) -> Optional[float]: band = self.tolerance() @@ -265,6 +427,13 @@ def refusal(self) -> Optional[str]: f"'{self.operation}' has no recorded demonstration; there is no value and no " "tolerance, and nothing here says what performing it well would look like" ) + if n >= MIN_DEMONSTRATIONS: + return ( + f"'{self.operation}' has {n} demonstration(s) of {self.metric} and every one reads " + f"{self.demonstrations[0].value} to full precision; independent runs of a continuous " + "quantity do not agree exactly, so this is a transcribed number rather than a " + "demonstrated range" + ) return ( f"'{self.operation}' has {n} demonstration(s) of {self.metric}; a tolerance needs " f"{MIN_DEMONSTRATIONS}. The value is established and the spread is not, so no envelope " @@ -277,6 +446,10 @@ def caveat(self) -> Optional[str]: A range from one demonstrator describes that demonstrator. It is still the best evidence available and it is not evidence about the operation, so the caveat travels with every parity claim made against it rather than living in a methods section. + + There is deliberately no mixed-conditions branch. The constructor refuses that envelope + outright, and a caveat that can never fire would be a second statement of a rule the + constructor already owns. """ if self.tolerance() is None: return None @@ -301,6 +474,13 @@ class Parity: `expert_n` and `machine_n` are carried because the attainment is meaningless without them and they are the first thing dropped when a result is summarized. MEETS over three and three is a finding; MEETS over three and one is a bug this module refuses to produce. + + `matched_n` is separate from `machine_n` because they answer different questions and + collapsing them erases runs. `machine_n` counts the observations handed in; `matched_n` + counts the ones measured under the conditions the expert demonstrated. Reporting a + condition mismatch as machine_n=0 would make a report that discarded twenty runs + indistinguishable from one that never had any, and the difference between those is whether + somebody has work to re-file or work to do. """ operation: str @@ -308,6 +488,7 @@ class Parity: reason: str expert_n: int machine_n: int + matched_n: int @property def demonstrated(self) -> bool: @@ -324,6 +505,8 @@ def attainment( The order of the checks is the substance of the function, because every one of them is a path by which a comparison with too little behind it becomes a MEETS: + an observation of another quantity refused, not compared; raises + the same observation entered twice refused, not counted; raises no envelope, or no demonstrations nothing to attain; UNMEASURED no machine observations nothing to place; UNMEASURED none under the demonstrated conditions measured, but not of the same experiment @@ -331,13 +514,18 @@ def attainment( too few machine observations a number that cannot be told from luck otherwise compare the machine's WORST run to the envelope + The first two raise rather than returning a verdict, matching `Envelope.__post_init__`. + They are not states of the evidence, they are the wrong evidence, and a function that + returned UNMEASURED for them would leave a recovery figure logged in percent sitting in a + report as a row somebody might later fill in. + Conditions are matched by exact string equality, which is crude on purpose. The alternative to a blunt match is no match, and no match compares a picogram run against a nanogram demonstration and reports parity. """ # The untaught case -- no envelope AND no observations -- is the common one, and it is # exactly the case where neither argument carries the operation's name. Without the - # caller supplying it the refusal reads "no expert has demonstrated 'unknown'", which + # caller supplying it the refusal reads "no expert envelope places 'unknown'", which # names nothing and is useless as a queue entry. op = ( envelope.operation @@ -345,16 +533,39 @@ def attainment( else (observations[0].operation if observations else (operation or "unknown")) ) + if envelope is not None: + for o in observations: + if ( + o.operation != envelope.operation + or o.metric != envelope.metric + or o.units != envelope.units + ): + raise ValueError( + f"envelope for '{envelope.operation}' measures {envelope.metric} in " + f"{envelope.units}, but a machine observation by {o.by} records {o.metric} in " + f"{o.units} for '{o.operation}'; two metrics pooled into one comparison is a " + "number about nothing, and the machine side is where that number enters" + ) + repeats = _repeats(observations) + if repeats: + first = repeats[0] + raise ValueError( + f"{len(repeats)} machine observation(s) of '{op}' are identical to an earlier entry " + f"(by {first.by}, value {first.value}, evidence '{first.evidence}'); one run counted " + f"{MIN_DEMONSTRATIONS} times is one run, and the minimum exists to refuse it" + ) + if envelope is None or envelope.n() == 0: return Parity( operation=op, attainment=Attainment.UNMEASURED, reason=( - f"no expert has demonstrated '{op}'; there is no envelope to place a machine " - "against, and a machine number with nothing to compare it to is not a result" + f"no expert envelope places '{op}'; there is no demonstrated range to attain, and a " + "machine number with nothing to compare it to is not a result" ), expert_n=0 if envelope is None else envelope.n(), machine_n=len(observations), + matched_n=0, ) if not observations: @@ -367,6 +578,7 @@ def attainment( ), expert_n=envelope.n(), machine_n=0, + matched_n=0, ) demonstrated_conditions = envelope.conditions() @@ -381,7 +593,8 @@ def attainment( "the machine is unmeasured" ), expert_n=envelope.n(), - machine_n=0, + machine_n=len(observations), + matched_n=0, ) band = envelope.tolerance() @@ -394,7 +607,8 @@ def attainment( f"no tolerance: {envelope.refusal()}" ), expert_n=envelope.n(), - machine_n=len(matching), + machine_n=len(observations), + matched_n=len(matching), ) if len(matching) < MIN_DEMONSTRATIONS: @@ -407,21 +621,31 @@ def attainment( "A run that landed inside the range cannot be told from one that got lucky" ), expert_n=envelope.n(), - machine_n=len(matching), + machine_n=len(observations), + matched_n=len(matching), ) values = [o.value for o in matching] low, high = min(values), max(values) lo, hi = band - if envelope.goal is Goal.HIGHER: + # Units come off the observation, not off the envelope. The guard above has already made + # them equal; reading them from the value's own record is what keeps a future mismatch out + # of the prose rather than hidden inside it. + units = matching[0].units + if envelope.goal.outside_is_always_worse: + meets = low >= lo and high <= hi + worst = f"runs span {low}-{high} {units} against a demonstrated {lo}-{hi}" + elif envelope.goal is Goal.HIGHER: meets = low >= lo - worst = f"worst run {low} {envelope.units} against a demonstrated floor of {lo}" + worst = f"worst run {low} {units} against a demonstrated floor of {lo}" elif envelope.goal is Goal.LOWER: meets = high <= hi - worst = f"worst run {high} {envelope.units} against a demonstrated ceiling of {hi}" + worst = f"worst run {high} {units} against a demonstrated ceiling of {hi}" else: - meets = low >= lo and high <= hi - worst = f"runs span {low}-{high} {envelope.units} against a demonstrated {lo}-{hi}" + raise ValueError( + f"'{envelope.goal}' is a Goal this comparison does not handle; a direction that falls " + "through to a default silently inherits some other metric's semantics" + ) caveat = envelope.caveat() tail = f". Caveat: {caveat}" if caveat else "" @@ -434,7 +658,8 @@ def attainment( f"{envelope.n()} demonstration(s): {worst}{tail}" ), expert_n=envelope.n(), - machine_n=len(matching), + machine_n=len(observations), + matched_n=len(matching), ) return Parity( operation=op, @@ -444,7 +669,8 @@ def attainment( f"{envelope.n()} demonstration(s): {worst}{tail}" ), expert_n=envelope.n(), - machine_n=len(matching), + machine_n=len(observations), + matched_n=len(matching), ) @@ -461,6 +687,18 @@ class Transferable: experiment with a metric, a unit, and a direction, which is the difference between a backlog and a wish. + `measurement` keys into `qc.MEASUREMENTS` wherever the quantity is one this package + already names. `qc` owns that definition and its units; restating them here as free + strings would be two statements of one quantity, and the two drift the first time either + is edited. Where the key is set, `metric` and `units` are checked against the resolved + `Measurement` and a disagreement is refused. `enforced_criteria()` then resolves the + thresholds a gate will actually apply to it, which is the number with a value on it. + + `basis` records where the CHOICE OF METRIC came from. A metric choice is a scientific + claim and this package treats those as attributable everywhere else -- `qc.Criterion`, + `intelligence.Judgment`, `throughput.Duration` all carry one. The uncomfortable answer + here is INTUITION for every entry, and recording that is what makes them arguable. + `why_this_metric` is prose and no checker reads it. It is here because the wrong metric is the failure that survives every other check in this module: a demonstration measured on something that does not decide the outcome produces a real envelope, a real attainment, @@ -474,22 +712,60 @@ class Transferable: conditions: str # what a demonstration must hold constant for it to transfer demonstrator: str # the kind of expert whose performance is the standard why_this_metric: str + basis: Basis # where the choice of metric came from, not where a threshold came from + measurement: Optional[str] = None # key into qc.MEASUREMENTS, where qc names this quantity note: str = "" + def __post_init__(self) -> None: + if self.measurement is None: + return + known = MEASUREMENTS.get(self.measurement) + if known is None: + raise ValueError( + f"'{self.op}' claims to measure qc measurement '{self.measurement}', which qc does " + "not define; a key that resolves to nothing is a restatement wearing a link" + ) + if self.units != known.units or self.metric != known.note: + raise ValueError( + f"'{self.op}' states {self.metric} in {self.units}; qc's '{self.measurement}' is " + f"{known.note} in {known.units}. qc owns this quantity, and two statements of one " + "quantity eventually disagree with nothing to reveal which is wrong" + ) + + def enforced_criteria(self) -> Tuple[Criterion, ...]: + """The asserted thresholds a gate will apply to this metric, resolved from `qc`. + + Any envelope this module eventually produces gets judged against these, so a report + showing the demonstrated range beside `intelligence`'s benchmark and omitting them would + omit the only asserted number with a value on it -- the one that actually stops a run. + """ + if self.measurement is None: + return () + out: List[Criterion] = [] + for gate in QC_GATES.values(): + for c in gate.criteria: + if c.measurement == self.measurement: + out.append(c) + return tuple(out) + # The operations in the reference protocols where an expert's performance is the standard # and no protocol text captures it. Deliberately not every step: a step whose outcome is set -# by the instrument rather than by the person -- a thermal hold, a run-folder read -- has -# nothing to transfer, and listing it would inflate the backlog with work no demonstration -# would settle. +# by the instrument rather than by the person has nothing to transfer. That exclusion is +# written out in EXEMPT below rather than left to this tuple's absences, because an absence +# and an exemption have the same shape and make opposite claims. TRANSFERABLE: Tuple[Transferable, ...] = ( Transferable( op="start_sort", - metric="fraction of target wells containing exactly one cell", - units="fraction", + # Resolved from qc rather than restated. qc.MEASUREMENTS owns this quantity, and the + # gate that will enforce a threshold on it reads the same key. + metric=MEASUREMENTS["well_occupancy"].note, + units=MEASUREMENTS["well_occupancy"].units, + measurement="well_occupancy", goal=Goal.HIGHER, conditions="a known cell line at the working suspension density, 96-well plate, one cell per well", demonstrator="a scientist who runs the sorter routinely and adjusts the gate by eye", + basis=Basis.INTUITION, why_this_metric=( "occupancy is what the rest of the run is spent on. Everything downstream costs a " "reagent set per well whether or not a cell landed in it, so the operation's value is " @@ -510,6 +786,7 @@ class Transferable: goal=Goal.WINDOW, conditions="the working volume of this prep, the tips actually used, a full plate", demonstrator="a scientist pipetting the same addition by hand", + basis=Basis.INTUITION, why_this_metric=( "at picogram input a volume error is an efficiency change rather than a rounding, and " "it biases the dataset instead of breaking it. Two sided because delivering more than " @@ -527,6 +804,7 @@ class Transferable: goal=Goal.HIGHER, conditions="a spike-in of known quantity, the same bead ratio, the same elution volume", demonstrator="a scientist who has done this cleanup enough to read a pellet", + basis=Basis.INTUITION, why_this_metric=( "recovery is the only quantity that distinguishes a cleanup that worked from one that " "removed the library along with the supernatant, and both produce a plate that looks " @@ -545,6 +823,7 @@ class Transferable: goal=Goal.LOWER, conditions="the same number of wells, normalized from the same quantification method", demonstrator="a scientist normalizing and pooling by hand from a quant plate", + basis=Basis.INTUITION, why_this_metric=( "pooling is where a plate stops being 96 samples and becomes one tube. Uneven " "representation is not recoverable afterward at any read depth, and it is invisible " @@ -558,6 +837,7 @@ class Transferable: goal=Goal.HIGHER, conditions="the same seal, the same block, the same program, door closed", demonstrator="a scientist sealing and loading the plate", + basis=Basis.INTUITION, why_this_metric=( "the thermal program belongs to the instrument; what a person contributes at this step " "is the seal, and evaporation is what a bad one costs. Measuring block uniformity here " @@ -576,6 +856,7 @@ class Transferable: goal=Goal.WINDOW, conditions="the same dilution factor, the same plate, the same tip type", demonstrator="a scientist performing the serial dilution by hand", + basis=Basis.INTUITION, why_this_metric=( "a serial dilution compounds its own error: a systematic offset at the first transfer " "is present in every subsequent one, so the metric has to be the delivered volume " @@ -588,6 +869,44 @@ class Transferable: TRANSFERABLE_BY_OP: Dict[str, Transferable] = {t.op: t for t in TRANSFERABLE} +# Operations a protocol runs where nothing transfers from a person, with the reason why. +# This table exists so that "no Transferable" stops being one state covering two claims. +# +# The entries are deliberately narrow, because a table that exempts an operation is a table +# that removes it from a backlog, and a generous one would be the author-settable field that +# silences a report. Writing a parameter, issuing a start, reading a status, and checking a +# control link are operations whose outcome the instrument decides; a person does not perform +# them better. Anything a person does with their hands -- loading material, priming fluidics +# -- is NOT here, because the honest answer for those is that nobody has yet said what to +# measure, and that answer belongs in the queue. +EXEMPT: Dict[str, str] = { + "discover_usb": "a control-link check; it reports whether the path enumerates, and nobody enumerates it better", + "probe_http": "a control-link check against the instrument's API; the answer is the instrument's", + "probe_tcp": "a control-link check against the instrument's socket; the answer is the instrument's", + "load_protocol": "transferring a file to the instrument; the file's contents are the expert work, not the transfer", + "upload_program": "transferring a file to the instrument; the file's contents are the expert work, not the transfer", + "upload_manifest": "transferring a file to the sequencer; the manifest's contents are the expert work, not the transfer", + "select_program": "naming a stored program; the choice is made upstream and this step is a write", + "set_deposition": "writing a dispense parameter; the value is the expert decision and it is made before this step", + "set_run_parameters": "writing run parameters to the sequencer; the values are decided upstream", + "set_temperature": "writing a setpoint; holding it is the instrument's job and its uniformity is a benchmark", + "set_injection": "writing injection parameters; the method is the expert work and it is written before this step", + "start_run": "issuing a start; what follows is the instrument's run, on parameters already written", + "start_method": "issuing a start; what follows is the instrument's method, on a setpoint already written", + "wait_complete": "waiting for the instrument to report done; there is no performance here to measure", + "get_status": "reading state off the instrument; the number is the instrument's", + "watch_run_folder": "reading state off a folder the sequencer writes; the number is the sequencer's", + "read_absorbance": "the instrument's own measurement; whether it can be trusted is a benchmark in `intelligence`, not a demonstration", +} + +_BOTH = sorted(set(EXEMPT) & set(TRANSFERABLE_BY_OP)) +if _BOTH: + raise ValueError( + f"{_BOTH} are declared both transferable and exempt; an operation cannot both need a " + "demonstration and have nothing to demonstrate" + ) + + # Every expert demonstration recorded in this repo. There are none. # # This is the finding, not a placeholder. Six instruments have been reverse-engineered, two @@ -608,12 +927,19 @@ class Transferable: MACHINE_OBSERVATIONS: Tuple[MachineObservation, ...] = () -def _build_envelopes() -> Dict[str, Envelope]: - by_op: Dict[str, List[Demonstration]] = {} +def _build_envelopes() -> Dict[Tuple[str, str], Envelope]: + """One envelope per (operation, conditions). + + Keyed on both because an envelope is a range over one experiment. Grouping on the + operation alone would put a nanogram demonstration and a picogram demonstration into one + range -- the comparison `attainment` refuses on the machine side, handed to it on the + expert side already pooled and no longer visible as two experiments. + """ + by_key: Dict[Tuple[str, str], List[Demonstration]] = {} for d in DEMONSTRATIONS: - by_op.setdefault(d.operation, []).append(d) - out: Dict[str, Envelope] = {} - for op, demos in by_op.items(): + by_key.setdefault((d.operation, d.conditions), []).append(d) + out: Dict[Tuple[str, str], Envelope] = {} + for (op, conditions), demos in by_key.items(): spec = TRANSFERABLE_BY_OP.get(op) if spec is None: raise ValueError( @@ -621,7 +947,7 @@ def _build_envelopes() -> Dict[str, Envelope]: "measures; a demonstration whose metric nobody specified cannot be compared to " "anything" ) - out[op] = Envelope( + out[(op, conditions)] = Envelope( operation=op, metric=spec.metric, units=spec.units, @@ -631,12 +957,26 @@ def _build_envelopes() -> Dict[str, Envelope]: return out -ENVELOPES: Dict[str, Envelope] = _build_envelopes() +ENVELOPES: Dict[Tuple[str, str], Envelope] = _build_envelopes() -def envelope_for(operation: str) -> Optional[Envelope]: - """The expert envelope for an operation, or None where nobody has demonstrated it.""" - return ENVELOPES.get(operation) +def envelopes_for(operation: str) -> Tuple[Envelope, ...]: + """Every envelope recorded for an operation, one per set of conditions.""" + return tuple(env for (op, _conditions), env in ENVELOPES.items() if op == operation) + + +def envelope_for(operation: str, conditions: Optional[str] = None) -> Optional[Envelope]: + """The expert envelope for an operation, or None where nobody has demonstrated it. + + With `conditions` unset this answers only where the operation has been demonstrated under + exactly one set of them. Two sets are two experiments, and "the envelope for this + operation" is then a question with no answer rather than a choice this function should + quietly make on the caller's behalf. `taught()` says which of the two it is in words. + """ + if conditions is not None: + return ENVELOPES.get((operation, conditions)) + found = envelopes_for(operation) + return found[0] if len(found) == 1 else None def observations_for(operation: str) -> Tuple[MachineObservation, ...]: @@ -644,10 +984,18 @@ def observations_for(operation: str) -> Tuple[MachineObservation, ...]: def demonstrations_still_needed(operation: str) -> int: - """How many more expert demonstrations before this operation has an envelope.""" - env = envelope_for(operation) - have = 0 if env is None else env.n() - return max(0, MIN_DEMONSTRATIONS - have) + """How many more expert demonstrations before this operation has an envelope. + + Never zero while `taught()` reports untaught. The two get read side by side, and a backlog + saying nothing is outstanding beside a verdict saying nothing is established is a + disagreement no reader can resolve. + """ + best = 0 + for env in envelopes_for(operation): + if env.tolerance() is not None: + return 0 + best = max(best, env.n()) + return max(1, MIN_DEMONSTRATIONS - best) def taught(operation: str) -> Tuple[bool, str]: @@ -657,10 +1005,26 @@ def taught(operation: str) -> Tuple[bool, str]: and a missing measurement is not a met criterion. An operation nobody has demonstrated is untaught, and an operation demonstrated once is still untaught: something was shown, and what was shown carries no tolerance, so there is nothing for a machine to be held to. + + An exempt operation is untaught too. Exemption says no demonstration is coming, not that + one has happened, and reporting it as taught would be the author-settable field that + quiets a report. """ env = envelope_for(operation) spec = TRANSFERABLE_BY_OP.get(operation) if env is None: + several = envelopes_for(operation) + if len(several) > 1: + return False, ( + f"'{operation}' has been demonstrated under {len(several)} different sets of " + "conditions; each is its own envelope and none of them is the envelope for the " + "operation, so there is no single range to hold a machine to" + ) + if operation in EXEMPT: + return False, ( + f"'{operation}' is exempt from demonstration: {EXEMPT[operation]}. Nothing is being " + "claimed about an expert's performance of it, and nothing will be" + ) if spec is None: return False, ( f"no expert demonstration of '{operation}' is recorded and none is specified; " @@ -678,7 +1042,53 @@ def taught(operation: str) -> Tuple[bool, str]: ) -def untaught_operations(protocol) -> List[Tuple[str, str]]: +class _Counted: + """A result that reports how many things it examined, not only what it found. + + An empty list is the same object whether nothing was checked or everything passed, and no + reader can tell those apart -- which is how a report over zero operations reads as a lab + that has finished teaching its machines. These results behave like the sequence they + replace, so a caller that only wanted the rows keeps working, and they carry the count for + the caller that has to tell a refusal from a clean bill. + """ + + def _rows(self) -> Tuple: + raise NotImplementedError + + def __bool__(self) -> bool: + return bool(self._rows()) + + def __len__(self) -> int: + return len(self._rows()) + + def __iter__(self): + return iter(self._rows()) + + def __getitem__(self, index): + return self._rows()[index] + + +@dataclass(frozen=True) +class UntaughtOperations(_Counted): + """Every operation of one protocol with no expert envelope, and how many were examined.""" + + rows: Tuple[Tuple[str, str], ...] + operations_considered: int + + def _rows(self) -> Tuple: + return self.rows + + def refusal(self) -> Optional[str]: + """Why this result says nothing, or None where it says something.""" + if self.operations_considered: + return None + return ( + "no operation was examined; this protocol declares no steps, so an empty result means " + "nothing was checked rather than nothing is untaught" + ) + + +def untaught_operations(protocol) -> UntaughtOperations: """(op, reason) for every distinct operation in a protocol with no expert envelope. Deliberately overlaps `intelligence.untrusted_ops` and answers a different question. That @@ -688,15 +1098,15 @@ def untaught_operations(protocol) -> List[Tuple[str, str]]: instrument, the other is bench time by the scientist, and the second cannot be bought. """ out: List[Tuple[str, str]] = [] - seen = set() + seen: List[str] = [] for step in protocol.steps: if step.op in seen: continue - seen.add(step.op) + seen.append(step.op) ok, why = taught(step.op) if not ok: out.append((step.op, why)) - return out + return UntaughtOperations(rows=tuple(out), operations_considered=len(seen)) # -- the transfer report ------------------------------------------------------- @@ -706,10 +1116,12 @@ def untaught_operations(protocol) -> List[Tuple[str, str]]: class TransferRow: """One operation: what the expert established, what the machine did, and whether it attains. - `benchmarks` is resolved from `intelligence` rather than restated, so this report cannot - disagree with that layer about what was asserted. The row carries both because the whole - point is that they are different evidence: side by side, an operation with a benchmark and - no envelope is visibly a target nobody has shown is reachable. + `benchmarks` is resolved from `intelligence` and `criteria` from `qc` rather than either + being restated, so this report cannot disagree with those layers about what was asserted. + The row carries all three because the whole point is that they are different evidence: + side by side, an operation with an unmet benchmark and no envelope is visibly a target + nobody has shown is reachable, and an operation with a qc criterion and no envelope is a + threshold a gate will enforce against a range that does not exist. """ operation: str @@ -717,16 +1129,45 @@ class TransferRow: observations: Tuple[MachineObservation, ...] parity: Parity benchmarks: Tuple[Benchmark, ...] + criteria: Tuple[Criterion, ...] specified: bool # a Transferable declares what a demonstration would measure @property def has_expert_envelope(self) -> bool: return self.envelope is not None and self.envelope.tolerance() is not None + @property + def unmet_benchmarks(self) -> Tuple[Benchmark, ...]: + """Benchmarks `intelligence` has not recorded as met. Resolved there, not recomputed.""" + return tuple(b for b in self.benchmarks if not b.status.trusted) + + @property + def met_benchmarks(self) -> Tuple[Benchmark, ...]: + return tuple(b for b in self.benchmarks if b.status.trusted) + @property def asserted_only(self) -> bool: - """A target somebody asserted, and nobody has demonstrated is reachable.""" - return bool(self.benchmarks) and not self.has_expert_envelope + """A target somebody asserted, nobody has met, and nobody has demonstrated is reachable. + + The status is the whole point and it is why `intelligence.BenchmarkStatus` exists. A + benchmark that was MEASURED and PASSED, with a counted series behind it, is not a target + nobody has hit, and filing it here would state the opposite of the evidence. + """ + return bool(self.unmet_benchmarks or self.criteria) and not self.has_expert_envelope + + @property + def met_without_envelope(self) -> bool: + """Every asserted target met, and no expert range to place the machine in. + + Its own state rather than a shade of `asserted_only`: something was measured and it + passed, and what is missing is the demonstration that would say whether the target was + the right one. + """ + return ( + bool(self.met_benchmarks) + and not self.unmet_benchmarks + and not self.has_expert_envelope + ) @dataclass @@ -741,14 +1182,28 @@ def untaught(self) -> List[TransferRow]: def attained(self) -> List[TransferRow]: return [r for r in self.rows if r.parity.demonstrated] + def below(self) -> List[TransferRow]: + """Operations the machine performs WORSE than the person, on enough runs to say so. + + The one verdict in this module that is a finding against the machine, and it had no + accessor and no count. A bucket nobody can read out is a bucket that never fires. + """ + return [r for r in self.rows if r.parity.attainment is Attainment.BELOW] + + def unmeasured(self) -> List[TransferRow]: + return [r for r in self.rows if r.parity.attainment is Attainment.UNMEASURED] + def asserted_only(self) -> List[TransferRow]: - """Operations with a benchmark and no envelope. + """Operations with an unmet assertion and no envelope. The most common state in a real programme and the one that reads as progress: a number exists, it is written down, it has a unit, and nobody has shown a person can hit it. """ return [r for r in self.rows if r.asserted_only] + def met_without_envelope(self) -> List[TransferRow]: + return [r for r in self.rows if r.met_without_envelope] + def unplaceable(self) -> List[TransferRow]: """Operations where numbers exist on both sides and still say nothing.""" return [ @@ -757,6 +1212,16 @@ def unplaceable(self) -> List[TransferRow]: if r.parity.attainment is Attainment.INDISTINGUISHABLE_FROM_UNMEASURED ] + def discarded_observations(self) -> int: + """Machine runs that exist and sit under conditions no expert demonstrated. + + Carried because a row whose runs were all discarded reports UNMEASURED, which is the + same word a row with no runs at all gets. The count is what separates them, and + `conditions` is free text on both sides, so relabelling one string is otherwise enough + to make a BELOW verdict disappear without trace. + """ + return sum(r.parity.machine_n - r.parity.matched_n for r in self.rows) + def transfers(self) -> bool: """True only if every operation in this report has an envelope the machine attains. @@ -766,14 +1231,33 @@ def transfers(self) -> bool: """ return bool(self.rows) and all(r.parity.demonstrated for r in self.rows) + def refusal(self) -> Optional[str]: + """Why this report says nothing, or None where it says something.""" + if self.rows: + return None + return ( + "no operation was examined; every count here is a zero about no lab, and an empty " + "report is not a lab with nothing left to teach" + ) + def counts(self) -> Dict[str, int]: + """One key per attainment, so the four verdict buckets sum to `operations`. + + Without `below` and `unmeasured` the row saying the machine performs worse than the + person was invisible in every number this report produced. A set of buckets that does + not sum is a set of buckets with somewhere for a verdict to go missing. + """ return { "operations": len(self.rows), "specified": sum(1 for r in self.rows if r.specified), "with_envelope": sum(1 for r in self.rows if r.has_expert_envelope), "attained": len(self.attained()), + "below": len(self.below()), + "unmeasured": len(self.unmeasured()), "unplaceable": len(self.unplaceable()), "asserted_only": len(self.asserted_only()), + "met_without_envelope": len(self.met_without_envelope()), + "discarded_observations": self.discarded_observations(), } @@ -782,7 +1266,8 @@ def transfer_report(operations: Sequence[str]) -> TransferReport: Duplicate operations collapse, keeping the given order. Everything on a row is resolved from the layer that owns it -- envelopes from the demonstrations, benchmarks from - `intelligence` -- so a row cannot state a target this package states differently elsewhere. + `intelligence`, enforced thresholds from `qc` -- so a row cannot state a target this + package states differently elsewhere. """ rows: List[TransferRow] = [] seen = set() @@ -790,8 +1275,16 @@ def transfer_report(operations: Sequence[str]) -> TransferReport: if op in seen: continue seen.add(op) - env = envelope_for(op) obs = observations_for(op) + env = envelope_for(op) + if env is None: + # Several sets of conditions: place the machine against the envelope for the + # conditions it was actually run under, and against nothing where its runs span more + # than one. Picking one for the caller is what reports a picogram run as parity. + conditions = {o.conditions for o in obs} + if len(conditions) == 1: + env = envelope_for(op, conditions=conditions.pop()) + spec = TRANSFERABLE_BY_OP.get(op) rows.append( TransferRow( operation=op, @@ -799,7 +1292,8 @@ def transfer_report(operations: Sequence[str]) -> TransferReport: observations=obs, parity=attainment(env, obs, operation=op), benchmarks=tuple(BENCHMARKS_BY_OP.get(op, ())), - specified=op in TRANSFERABLE_BY_OP, + criteria=() if spec is None else spec.enforced_criteria(), + specified=spec is not None, ) ) return TransferReport(rows=rows) @@ -814,9 +1308,17 @@ class NextDemonstration: instrument: str operations: Tuple[str, ...] - demonstrations_needed: int steps_blocked: int + @property + def demonstrations_needed(self) -> int: + """Derived from the operations rather than stored beside them. + + A stored count is a field an author can set to disagree with the list it counts, and + nothing in the row would reveal which of the two was wrong. + """ + return sum(demonstrations_still_needed(op) for op in self.operations) + @property def cost(self) -> int: """Demonstrations to perform. A crude proxy for the expert's time, and an honest one: @@ -824,7 +1326,59 @@ def cost(self) -> int: return self.demonstrations_needed -def demonstration_queue(protocols: Sequence) -> List[NextDemonstration]: +@dataclass(frozen=True) +class UnspecifiedOperation: + """An operation a protocol runs that neither TRANSFERABLE nor EXEMPT accounts for. + + Not a queue entry, because it cannot be scheduled: a demonstration needs a metric, a unit, + and a direction, and none of those exists yet. The work it names is deciding what to + measure, which is an afternoon at a desk rather than an afternoon at the bench with real + material, and keeping the two in separate buckets is what stops them being ranked against + each other. What it is not is absent. + """ + + instrument: str + operation: str + + @property + def demonstrations_needed(self) -> int: + return demonstrations_still_needed(self.operation) + + +@dataclass(frozen=True) +class DemonstrationQueue(_Counted): + """The SME queue, plus what it could not queue and why. + + `entries` on its own is a list that reads identically for "nothing to rank", "nothing was + handed to me", and "everything is taught". `operations_considered` separates the second + from the other two, and `unspecified` stops the third being reached by dropping work. + """ + + entries: Tuple[NextDemonstration, ...] + unspecified: Tuple[UnspecifiedOperation, ...] + exempt: Tuple[str, ...] + operations_considered: int + + def _rows(self) -> Tuple: + return self.entries + + def refusal(self) -> Optional[str]: + """Why this queue says nothing, or None where it says something.""" + if self.operations_considered: + return None + return ( + "no operation was examined; nothing was handed to this queue, and an empty queue over " + "no protocols is not a lab that has finished teaching its machines" + ) + + def cost(self) -> int: + """Demonstrations across the whole queue, including work nobody has specified yet.""" + return sum(e.cost for e in self.entries) + sum( + u.demonstrations_needed for u in self.unspecified + ) + + +def demonstration_queue(protocols: Sequence) -> DemonstrationQueue: """The SME queue, ranked by how many untaught operations one sitting would settle. Grouped by INSTRUMENT for the same reason `ledger.unlocks` is, and the reason is not the @@ -839,43 +1393,69 @@ def demonstration_queue(protocols: Sequence) -> List[NextDemonstration]: Scoped to the operations the given protocols actually contain, and the ranking is computed from them: hand it a different set of protocols and the order changes, because a station carrying operations in several flows is where a scientist's afternoon buys the most. + + An operation with no `Transferable` is not skipped. It goes to `unspecified` unless EXEMPT + accounts for it, because "nobody wrote a spec" and "nothing transfers here" are opposite + claims that a `continue` renders identical, and the first is the least taught state in the + protocol. """ ops_by_instrument: Dict[str, List[str]] = {} steps: Dict[str, int] = {} - needed: Dict[str, int] = {} + unspecified: List[UnspecifiedOperation] = [] + exempt: List[str] = [] + considered: List[str] = [] for protocol in protocols: for step in protocol.steps: - if step.op not in TRANSFERABLE_BY_OP: + if step.op not in considered: + considered.append(step.op) + if step.op in EXEMPT: + if step.op not in exempt: + exempt.append(step.op) continue ok, _why = taught(step.op) if ok: continue + if step.op not in TRANSFERABLE_BY_OP: + # Keyed by (instrument, op) rather than op alone, because this is a queue of + # sittings: the same unspecified operation at two stations is two of them. + entry = UnspecifiedOperation(instrument=step.instrument, operation=step.op) + if entry not in unspecified: + unspecified.append(entry) + continue ops = ops_by_instrument.setdefault(step.instrument, []) if step.op not in ops: ops.append(step.op) - needed[step.instrument] = needed.get(step.instrument, 0) + demonstrations_still_needed( - step.op - ) steps[step.instrument] = steps.get(step.instrument, 0) + 1 - out = [ + entries = [ NextDemonstration( instrument=key, operations=tuple(ops), - demonstrations_needed=needed.get(key, 0), steps_blocked=steps.get(key, 0), ) for key, ops in ops_by_instrument.items() ] - out.sort(key=lambda q: (-len(q.operations), -q.steps_blocked, q.cost, q.instrument)) - return out + entries.sort(key=lambda q: (-len(q.operations), -q.steps_blocked, q.cost, q.instrument)) + return DemonstrationQueue( + entries=tuple(entries), + unspecified=tuple(unspecified), + exempt=tuple(exempt), + operations_considered=len(considered), + ) def teaching_summary() -> Dict[str, int]: - """How much of the expert performance behind these protocols has been recorded.""" + """How much of the expert performance behind these protocols has been recorded. + + Counted over this module's own tables. What a given protocol contains that neither table + accounts for is protocol-scoped and lives on `DemonstrationQueue.unspecified`, because a + summary reaching for a global set of protocols would be answering about a lab nobody named + in the call. + """ return { "operations_specified": len(TRANSFERABLE), + "operations_exempt": len(EXEMPT), "demonstrations_recorded": len(DEMONSTRATIONS), "operations_with_envelope": sum( 1 for op in TRANSFERABLE_BY_OP if taught(op)[0] @@ -888,12 +1468,14 @@ def teaching_summary() -> Dict[str, int]: __all__ = [ "DEMONSTRATIONS", "ENVELOPES", + "EXEMPT", "MACHINE_OBSERVATIONS", "MIN_DEMONSTRATIONS", "TRANSFERABLE", "TRANSFERABLE_BY_OP", "Attainment", "Demonstration", + "DemonstrationQueue", "Envelope", "Goal", "MachineObservation", @@ -902,10 +1484,13 @@ def teaching_summary() -> Dict[str, int]: "Transferable", "TransferReport", "TransferRow", + "UnspecifiedOperation", + "UntaughtOperations", "attainment", "demonstration_queue", "demonstrations_still_needed", "envelope_for", + "envelopes_for", "observations_for", "taught", "teaching_summary", diff --git a/tests/test_coverage.py b/tests/test_coverage.py index 1402315..1859b09 100644 --- a/tests/test_coverage.py +++ b/tests/test_coverage.py @@ -14,26 +14,38 @@ a declared visual check with no camera behind it a gate that cannot be evaluated because its input comes from a broken instrument a gate that CAN be evaluated and reads the wrong assay for the sample + a gate that fires only with a human at the instrument, in a package about unattended runs + a measurement set that says nothing about the assay, where silence must not read as yes + a gate report carrying two rows for one gate, where row ORDER must not decide the answer a report over no rows at all, which is the vacuous pass in its purest form + a report with rows in it and nothing destructive among them, which is the same pass + wearing the only costume the row count does not catch a vision capability with every requirement met, which must STILL leave the INVISIBLE failures uncovered, because that boundary is physics and not budget + a partial purchase, which is what a real purchase is, whose unbought failures must not + fall between the lists and the one on the other side, which is the reason the module is worth having: a material-destroying failure with neither path must appear in the report every time, never -omitted and never softened. +omitted and never softened. The positive controls are here for the same reason -- a suite +that only ever asserts False proves nothing about the wiring that returns True. """ from __future__ import annotations import dataclasses +import itertools +from typing import Dict import pytest -from autonomous_lab import build_ledger, protocols +from autonomous_lab import build_ledger, protocols, vision +from autonomous_lab import coverage as coverage_module from autonomous_lab.coverage import ( Cover, CoverageReport, CoverageRow, + GateStanding, Uncovered, coverage_report, mandatory_gates, @@ -42,18 +54,27 @@ ) from autonomous_lab.qc import ( LIBRARY_QUANT_GATE, + MEASUREMENTS, SORT_OCCUPANCY_GATE, GateReadiness, GateReport, + Measurement, Readiness, gate_report, ) -from autonomous_lab.recovery import FAILURES_BY_NAME, Detection, Severity, recovery_report +from autonomous_lab.recovery import ( + FAILURES_BY_NAME, + Detection, + Latency, + Severity, + recovery_report, +) from autonomous_lab.vision import ( CHECKS_BY_NAME, Observable, VisionCapability, VisionRequirement, + VisualCheck, check_catching, ) @@ -69,19 +90,19 @@ def _genomics(): return protocol, ledger, gate_report(protocol.name, ledger) -def _report(capability=None, gates=None) -> CoverageReport: +def _report(capability=None, gates=None, measurements=None) -> CoverageReport: """The composition over the genomics loop, at a stated vision capability. - The recovery report is rebuilt against the same capability every time. That is not - ceremony: coverage_report refuses a recovery report built against a different one, and it - refuses it because a mismatch produces rows that are individually well-formed and describe - a workcell nobody has. + The recovery report is rebuilt against the same capability AND the same gates every time. + That is not ceremony: coverage_report refuses a recovery report built against either a + different capability or different gates, and it refuses both because a mismatch produces + rows that are individually well-formed and describe a workcell nobody has. """ cap = capability or VisionCapability.none() protocol, ledger, real_gates = _genomics() gates = real_gates if gates is None else gates return coverage_report( - protocol, ledger, gates, recovery_report(protocol, gates, cap), cap + protocol, ledger, gates, recovery_report(protocol, gates, cap), cap, measurements ) @@ -99,6 +120,30 @@ def _gates_where(*readiness_rows) -> GateReport: return GateReport(protocol="single_cell_genomics", rows=list(readiness_rows)) +def _repaired_reader() -> GateReport: + """The library-quant gate, evaluable. The one repair that would change this report.""" + return _gates_where( + GateReadiness(LIBRARY_QUANT_GATE, Readiness.READY, "pretend the reader was fixed today") + ) + + +def _appropriate_measurements() -> Dict[str, Measurement]: + """MEASUREMENTS with the A260 read declared appropriate for a single-cell library. + + A scientific claim this repo does not make and no bench here supports. It exists so the + gate arm can be tested in the direction where it says yes -- without it, every assertion + about an appropriate assay is an assertion about a branch nothing reaches. + """ + return { + **MEASUREMENTS, + "library_conc_od": dataclasses.replace( + MEASUREMENTS["library_conc_od"], + instrument_appropriate=True, + inappropriate_reason="", + ), + } + + # -- the vacuous pass, in each of its costumes --------------------------------- @@ -450,7 +495,16 @@ def test_coverage_is_derived_and_never_a_stored_field(): eventually disagrees with them, and nothing reveals which one is wrong. """ stored = {f.name for f in dataclasses.fields(CoverageRow)} - for derived in ("cover", "uncovered", "severity", "observable", "destroys_material"): + for derived in ( + "cover", + "uncovered", + "severity", + "observable", + "destroys_material", + # One reading of the six-valued standing. Stored beside it, the two eventually disagree + # about whether a supervised or unmeasured gate counts, and nothing says which is wrong. + "gate_evaluable", + ): assert derived not in stored, f"'{derived}' is derivable and must not be storable" @@ -493,3 +547,372 @@ def test_the_counts_add_up(): assert counts["fixable"] + counts["structural"] == counts["uncovered"] assert counts["unwatched"] <= counts["uncovered"] assert counts["uncovered_destructive"] <= counts["uncovered"] + + +# -- the measurement set is an input, and silence in it is not a pass ---------- + + +def test_a_gate_reading_an_undeclared_measurement_is_not_coverage(): + """The fail-open seam: `measurements` is a parameter, and saying nothing is not saying yes. + + `qc.GateReport.inappropriate` skips a key it cannot resolve, which is right for the + question it answers -- it lists the assays known to be wrong -- and fail-open for the + question asked here. Hand this module a measurement set that describes no A260 read and + the wrong-assay arm has nothing to object to, so a gate whose assay nobody characterized + fills the covered column. That is qc's own empty-measurement-dict bug rebuilt one layer + up, and it is defeated by an omission rather than by a lie, which is what makes it quiet. + """ + repaired = _repaired_reader() + honest = _report(gates=repaired) + omits_the_assay = {k: v for k, v in MEASUREMENTS.items() if k != "library_conc_od"} + + for measurements in ({}, omits_the_assay): + report = _report(gates=repaired, measurements=measurements) + row = _row(report, "enzyme_activity_lost") + assert row.gate_standing is GateStanding.UNMEASURED + assert not row.gate_evaluable + assert row.cover is Cover.NEITHER + assert "not a declared measurement" in row.gate_reason + assert "unchecked assay" in row.gate_reason + + # The headline failure stays uncovered, and no count moves in the flattering direction. + assert not _row(report, "bead_pellet_aspirated").cover.covered + assert report.counts()["covered"] <= honest.counts()["covered"] + assert report.counts()["uncovered_destructive"] >= honest.counts()["uncovered_destructive"] + + +def test_a_demand_is_not_met_by_a_measurement_set_that_says_nothing(): + """The same omission through the other entry point, where it reads as compliance. + + `mandatory_gates` already refuses to treat a missing gate report as met. A measurement + set that mentions none of the gate's assays is the same skip wearing a different costume, + and it must land the same way. + """ + repaired = _repaired_reader() + demands = mandatory_gates(repaired, {}) + assert demands + assert all(not d.met for d in demands) + assert len(unmet_demands(repaired, {})) == len(demands) + + enzyme = next(d for d in demands if d.failure.name == "enzyme_activity_lost") + assert enzyme.gate == "library_quant_before_flow_cell" + assert "unchecked assay" in enzyme.reason + assert "is NOT met" in enzyme.statement() + + +def test_a_mandatory_gate_that_can_actually_fire_is_reported_met(): + """The positive control `Demand.met` never had, and the reason it was needed. + + Every assertion about `met` in this file is a negative, so rewriting the field to a + constant False leaves the suite green: the wiring from the gate arm into the demand, and + the filter `unmet_demands` applies over it, were untested in the only direction where + they can overclaim. enzyme_activity_lost is the one INVISIBLE material-destroying failure + that names a gate, so it is the only row that can carry this test. + """ + repaired = _repaired_reader() + measurements = _appropriate_measurements() + demands = mandatory_gates(repaired, measurements) + + enzyme = next(d for d in demands if d.failure.name == "enzyme_activity_lost") + assert enzyme.gate == "library_quant_before_flow_cell" + assert enzyme.met + assert "is met" in enzyme.statement() + + unmet = {d.failure.name for d in unmet_demands(repaired, measurements)} + assert "enzyme_activity_lost" not in unmet + assert unmet, "the rest name no assay at all and stand unmet" + + # And the same fact through the composition, so the two entry points cannot drift. + row = _row(_report(gates=repaired, measurements=measurements), "enzyme_activity_lost") + assert row.gate_standing is GateStanding.COUNTS + assert row.cover is Cover.GATE + + +# -- coverage means coverage with nobody in the room --------------------------- + + +def test_a_gate_that_needs_a_human_at_the_instrument_is_not_coverage(): + """`Readiness.evaluable` is true for SUPERVISED, and this package is about unattended runs. + + qc is right about its own question -- the number does arrive -- and reading its answer as + coverage counts the operator this whole package exists to remove. The path is not + hypothetical: `qc.readiness` maps a SUPERVISED step straight onto it. + """ + supervised = _gates_where( + GateReadiness( + SORT_OCCUPANCY_GATE, + Readiness.SUPERVISED, + "a human has to sit at the sorter and read occupancy off the screen", + ) + ) + row = _row(_report(gates=supervised), "empty_well_after_sort") + + assert row.gate_standing is GateStanding.SUPERVISED + assert not row.gate_evaluable + assert row.cover is Cover.NEITHER + assert "human at the instrument" in row.gate_reason + + # Refused as coverage and still named: a person does read that number, and claiming + # nobody would notice is the opposite overclaim. + assert row.residual is Detection.QC_GATE + assert not row.unwatched + + +def test_a_gate_path_this_module_refused_is_not_republished_as_a_residual(): + """One row must not assert a gate is not coverage and that the gate path still stands. + + `recovery` resolves the declared path against its own inputs and never asks whether the + number is about this sample, so a repaired reader gives it a live QC_GATE for a gate this + module has just shown reads the wrong assay. Carrying that answer through to `residual` + puts two computations of one fact on one row, and drops the failure out of `unwatched()` + -- the list the report tells a reader to read first. + """ + report = _report(gates=_repaired_reader()) + row = _row(report, "bead_pellet_aspirated") + + assert row.gate_standing is GateStanding.WRONG_ASSAY + assert row.effective.detection is Detection.QC_GATE + assert row.residual is None + assert row.unwatched + assert row in report.unwatched() + + +# -- an all-clear has to be an all-clear about something ----------------------- + + +def test_a_report_holding_nothing_destructive_is_not_complete(): + """The vacuity guard was on the row count, and the row count is not the population. + + `complete()` is the report's all-clear. Scope a report to rows that cost only time -- a + subprotocol, a filter, a one-step run -- and the destructive population is empty, nothing + is uncovered in it, and a lab with zero machine checks on anything earns the all-clear. + """ + report = _report() + quiet = CoverageReport( + protocol=report.protocol, + capability=report.capability, + rows=[r for r in report.rows if not r.destroys_material], + ) + assert quiet.rows, "the point is a report that was handed something" + assert quiet.counts()["covered"] == 0 + assert quiet.counts()["destructive"] == 0 + assert not quiet.in_scope() + assert not quiet.complete() + + one_row = CoverageReport( + protocol=report.protocol, + capability=report.capability, + rows=[_row(report, "tecan_read_timeout")], + ) + assert not one_row.complete() + + +# -- malformed and mismatched inputs, which is how one arm silences another ---- + + +def test_a_duplicate_gate_row_cannot_silence_an_unsatisfiable_one(): + """First-match resolution makes row ORDER an author-settable field that hides a dead gate. + + A report carrying two rows for one gate is malformed, and the failure mode is that an + optimistic duplicate placed above the real row wins. `qc.GateReport.unsatisfiable` still + names the gate on the very same object, so the composition would contradict its own + input. The appropriateness arm below it already iterates every matching row, so before + this was fixed one arm was duplicate-robust and the other was not. + """ + protocol, ledger, real_gates = _genomics() + dead = next(r for r in real_gates.rows if r.gate.name == SORT_OCCUPANCY_GATE.name) + optimistic = GateReadiness( + SORT_OCCUPANCY_GATE, Readiness.READY, "an optimistic duplicate row" + ) + capability = VisionCapability.none() + + for rows in ([dead, optimistic], [optimistic, dead]): + duplicated = GateReport(protocol=protocol.name, rows=list(rows)) + # Recovery is built against the honest single-row report on purpose: the malformed + # input under test is the gate report, not the composition of two reports. + report = coverage_report( + protocol, + ledger, + duplicated, + recovery_report(protocol, real_gates, capability), + capability, + ) + row = _row(report, "empty_well_after_sort") + + assert duplicated.unsatisfiable(), "the report says so about itself" + assert row.gate_standing is GateStanding.DEAD + assert row.cover is Cover.NEITHER + assert report.counts()["covered"] == 0 + + +def test_the_order_of_the_gate_rows_does_not_change_the_report(): + """Order is an input nothing in this suite varied, which is how a first-match bug lives. + + Every permutation of one malformed report has to produce one set of counts. A report + whose numbers depend on the order it was assembled in is not reporting a workcell. + """ + protocol, ledger, real_gates = _genomics() + dead = next(r for r in real_gates.rows if r.gate.name == SORT_OCCUPANCY_GATE.name) + rows = [ + dead, + GateReadiness(SORT_OCCUPANCY_GATE, Readiness.READY, "an optimistic duplicate row"), + GateReadiness(LIBRARY_QUANT_GATE, Readiness.READY, "pretend the reader was fixed today"), + ] + capability = VisionCapability.none() + consistent = GateReport(protocol=protocol.name, rows=[dead, rows[2]]) + measurements = _appropriate_measurements() + + seen = [] + for ordered in itertools.permutations(rows): + gates = GateReport(protocol=protocol.name, rows=list(ordered)) + report = coverage_report( + protocol, + ledger, + gates, + recovery_report(protocol, consistent, capability), + capability, + measurements, + ) + seen.append(report.counts()) + assert _row(report, "empty_well_after_sort").cover is Cover.NEITHER + assert all(counts == seen[0] for counts in seen) + + +def test_a_recovery_report_built_against_other_gates_is_refused(): + """The mismatch the vision axis was guarded against, on the axis that was not. + + Composing an optimistic gate report with a recovery report resolved against the real one + produces a row that says the gate is coverage and carries, on the same row, recovery's + finding that the gate is unsatisfiable. Both halves are well-formed. Together they + describe a workcell nobody has. + """ + protocol, ledger, real_gates = _genomics() + optimistic = _gates_where( + GateReadiness(SORT_OCCUPANCY_GATE, Readiness.READY, "pretend the sorter were decoded") + ) + capability = VisionCapability.none() + with pytest.raises(ValueError, match="disagree"): + coverage_report( + protocol, + ledger, + optimistic, + recovery_report(protocol, real_gates, capability), + capability, + ) + + +def test_a_recovery_report_that_never_saw_a_gate_report_is_refused(): + """`recovery_report(protocol)` takes every declared gate path on trust, and says so. + + Composing that with a real gate report is the quietest version of the mismatch, because + nobody had to assert anything false to produce it: the row reports the gate as dead and + the declared gate path as intact, and `unwatched()` loses the failures this package calls + the most destructive in the loop. + """ + protocol, ledger, gates = _genomics() + with pytest.raises(ValueError, match="disagree"): + coverage_report( + protocol, ledger, gates, recovery_report(protocol), VisionCapability.none() + ) + + +def test_a_visual_check_that_contradicts_the_failure_model_is_refused(monkeypatch): + """Observability is recorded twice, and this module is the only place the two records meet. + + `VisionCapability.available` tests the CHECK's observability; every consumer of the row + tests the FAILURE's. Declare a check that claims to see a denatured enzyme and the + physics ceiling stops being reported: the row comes back covered by a camera, and the + failure disappears from both of sota_lift's lists. + """ + claims_to_see_the_invisible = VisualCheck( + name="enzyme_activity_visible", + step_op="wgs_prep_lysis", + observes="a well of dead enzyme, which is pixel-for-pixel a well of live enzyme", + observable=Observable.VISIBLE, + catches="enzyme_activity_lost", + requires=(VisionRequirement.CAMERA,), + ) + monkeypatch.setattr(vision, "CHECKS", vision.CHECKS + (claims_to_see_the_invisible,)) + assert check_catching("enzyme_activity_lost") is claims_to_see_the_invisible + + with pytest.raises(ValueError, match="disagree"): + _report(_fully_equipped()) + + +# -- what a partial purchase does not buy -------------------------------------- + + +def test_sota_lift_names_what_a_partial_purchase_still_does_not_buy(): + """Two lists do not partition the uncovered set, and a real proposal is partial. + + Everything a lab can buy except a lighting rig lifts three failures and leaves the most + destructive routine failure in low-input prep exactly where it was. It is not at the + ceiling -- a camera sees a pellet -- so with two lists it appears in neither, and the + report becomes the purchase justification the second list was added to prevent. + """ + protocol, ledger, gates = _genomics() + everything_but_lighting = VisionCapability( + name="everything but lighting", + satisfied=tuple(r for r in VisionRequirement if r is not VisionRequirement.LIGHTING), + ) + lift = sota_lift(VisionCapability.none(), everything_but_lighting, protocol, ledger, gates) + + blocked = {b.failure.name: b for b in lift.still_blocked} + assert "bead_pellet_aspirated" in blocked + assert VisionRequirement.LIGHTING in blocked["bead_pellet_aspirated"].missing + assert "lighting" in blocked["bead_pellet_aspirated"].reason + assert lift.destructive_still_blocked(), "two of these destroy the sample" + + after = coverage_report( + protocol, + ledger, + gates, + recovery_report(protocol, gates, everything_but_lighting), + everything_but_lighting, + ) + uncovered = {row.failure.name for row in after.uncovered()} + named = {c.failure.name for c in lift.ceiling} | set(blocked) + assert named == uncovered, "every uncovered failure lands in exactly one list" + + counts = lift.counts() + assert counts["ceiling"] + counts["still_blocked"] == len(uncovered) + assert counts["still_blocked_destructive"] > 0 + assert set(blocked).isdisjoint({m.failure.name for m in lift.lifted}) + + +def test_nothing_is_still_blocked_once_every_requirement_is_bought(): + """The other direction, so the third list cannot be a bucket that always has rows in it.""" + protocol, ledger, gates = _genomics() + lift = sota_lift(VisionCapability.none(), _fully_equipped(), protocol, ledger, gates) + assert lift.still_blocked == [] + assert lift.destructive_ceiling(), "and the ceiling is untouched by any of it" + + +# -- the report says what it computes, and not more ----------------------------- + + +def test_the_module_claims_only_what_its_predicate_computes(): + """A NEITHER row means no detector and no gate, not that nothing in the lab would notice. + + The predicate is a deployed visual check or an evaluable gate. Two failures here are + raised by the instrument itself before anything moves -- machine-checkable, preventive, + and not counted by this module -- so a docstring claiming NEITHER means nothing + machine-checkable stands there prints a sentence the module's own rows contradict. + """ + report = _report() + for name in ("star_resource_busy", "tecan_read_timeout"): + row = _row(report, name) + assert row.cover is Cover.NEITHER + assert row.residual is Detection.INSTRUMENT + assert row.effective.latency is Latency.IMMEDIATE + + # The claim is the first sentence of each docstring, which is what a reader carries away. + # The paragraphs under it are free to explain why the narrower claim is the honest one. + summary = " ".join((Cover.covered.__doc__ or "").split()).split(".")[0] + assert "machine-checkable" not in summary + assert "visual check" in summary and "gate" in summary + + module_doc = " ".join((coverage_module.__doc__ or "").split()) + assert "nothing MACHINE-CHECKABLE stands between this failure" not in module_doc + assert "NO DECLARED DETECTOR AND NO EVALUABLE GATE" in module_doc + assert "residual" in module_doc diff --git a/tests/test_durability.py b/tests/test_durability.py index 24f9401..92c4f1e 100644 --- a/tests/test_durability.py +++ b/tests/test_durability.py @@ -10,6 +10,15 @@ The fifth test hunts the number. This module must not return a predicted failure rate under any name, because a reliability figure invented here would become a specification nobody measured. + +The later tests hunt the same four ways in through the doors an audit found open. A NaN in +a maintenance cell is the fifth way to arrive at "nobody counted it", and it used to be +read as a measurement. An obligation that recurs every inf days is the obligation that +recurs on nothing, spelled as a number. A campaign that lands exactly on a boundary was +cleared by a report whose own arithmetic calls the state it ends in lapsed. An uncounted +second unit erased a lapse the module had already proved. One service record filed under +nine keys entitled a workcell. And a guard installed at one of four entry points is a +guard at none, which is why the empty campaign is now checked at every door in one test. """ from __future__ import annotations @@ -162,6 +171,35 @@ def test_an_uncounted_unit_is_not_read_as_zero(): assert untracked.entitlement() is Entitlement.UNKNOWN +def test_a_nan_accrual_is_refused_rather_than_read_as_a_measurement(): + """The other missing-number sentinel, and the one a real loader produces. + + None is the module's uncounted marker and a blank maintenance cell round-tripped through + a spreadsheet or a JSON reader comes back as NaN, which is the same fact wearing a + different type. NaN compares False against every threshold in this module, so reading it + as a count reported a full budget, an ENTITLED instrument, and a clear campaign over a + cell nobody filled in -- absent data yielding a pass, in one call. + """ + for kwargs in ({"runs": float("nan")}, {"days": float("nan")}, {"hours": float("nan")}): + with pytest.raises(ValueError, match="means UNCOUNTED"): + Accrued(**kwargs) + with pytest.raises(ValueError, match="means UNCOUNTED"): + Accrued(runs=float("inf")) + assert Accrued(runs=None).charged(Charge.RUNS) is None, "None remains the way to say uncounted" + + +def test_a_negative_accrual_cannot_mint_budget_that_was_never_earned(): + """An instrument cannot have run minus nine hundred plates since it was serviced. + + Accepted, it added to the remaining budget: a hundred-run calibration reported a + thousand runs left and cleared a five-hundred-run campaign. + """ + with pytest.raises(ValueError, match="negative work"): + Accrued(runs=-900.0) + with pytest.raises(ValueError, match="negative work"): + Accrued(days=-1.0) + + # -- lapse ---------------------------------------------------------------------- @@ -193,6 +231,50 @@ def test_calendar_time_lapses_an_instrument_that_did_no_work(): assert idle.entitlement() is Entitlement.LAPSED +def test_a_proved_lapse_survives_an_uncounted_second_unit(): + """An uncounted unit must not erase a lapse the module has already proved. + + Checking `uncounted` before `exhausted` turned a definitively past-due calibration into + UNKNOWN the moment a second charge unit was added to the same interval, and the row left + `lapsed()`, `invalidating()`, and the boundary report with it. The lab was told to start + a counter; it was not told that five hundred days of results are unattributable. + """ + interval = _calibration(name="cal_days", every_days=365.0, every_runs=500.0) + health = _health(interval, since=Accrued(days=900.0)) # days counted, runs never + + assert health.entitlement() is Entitlement.LAPSED + assert len(health.lapsed()) == 1 + assert len(health.invalidating()) == 1, "the retrospective alarm must not go quiet" + row = health.lapsed()[0] + assert row.remaining.exhausted() == (Charge.DAYS,) + assert row.partial == (Charge.RUNS,), "the uncounted unit qualifies the finding, not the verdict" + assert "unattributable" in row.reason + assert "There is no budget to state" not in row.reason + assert interval.restores in row.restores and "count runs" in row.restores + + report = crosses_boundary(health, Campaign(days=90)) + assert report.lapsed(), "the lapse reaches the campaign report too" + assert report.uncertain(), "and the unit nobody counted is still an unavailable answer" + assert not report.clear + assert health.partially_counted() == (row,) + + +def test_a_reason_never_denies_a_budget_the_same_object_states(): + """Prose the module's own data contradicts is a claim, not a finding. + + An interval charged in two units with one of them counted has a stated remaining budget + on that unit, and the report said "There is no budget to state" beside it. + """ + interval = _calibration(every_runs=100.0, every_days=365.0) + health = _health(interval, since=Accrued(days=65.0)) # days counted, runs not + row = standing(health, interval) + assert row.entitlement is Entitlement.UNKNOWN, "an uncounted unit still blocks a clearance" + assert row.remaining.value(Charge.DAYS) == 300.0 + assert "300.0 days" in row.reason + assert "There is no budget to state" not in row.reason + assert row.partial == (Charge.RUNS,) + + def test_a_calibration_lapse_reaches_backwards_and_a_tip_box_does_not(): """The distinction that decides what a lapse costs. @@ -242,6 +324,76 @@ def test_a_campaign_inside_the_remaining_budget_is_clear(): assert report.crossings() == () +def test_a_campaign_that_lands_exactly_on_the_boundary_crosses_it(): + """The two boundary rules on `Remaining` have to be one rule, and the module says which. + + `exhausted()` calls a budget of exactly zero spent, not nearly spent. `crossed_by` used + a strict comparison, so a campaign consuming exactly the remaining budget was reported + clear -- and the state that campaign produces is one this same module calls LAPSED. The + last plate of the "clear" campaign is run by an instrument the report will not vouch + for. The existing tests straddle this case at 39 and 120 against 40 remaining. + """ + interval = _calibration(every_runs=100.0) + health = _health(interval, since=Accrued(runs=60.0)) # 40 left + + assert crosses_boundary(health, Campaign(runs=39)).clear, "one short of the boundary fits" + report = crosses_boundary(health, Campaign(runs=40)) + assert not report.clear + assert len(report.crossings()) == 1 + assert report.crossings()[0].entitlement is Entitlement.EXPIRING + assert report.crossings()[0].crossing == (Charge.RUNS,) + + ends_at = _health(interval, since=Accrued(runs=100.0)) + assert ends_at.entitlement() is Entitlement.LAPSED, "the state the cleared campaign ends in" + + +def test_an_empty_campaign_is_refused_at_every_door_that_accepts_one(): + """A guard on one of four entry points is a guard on none. + + The refusal lived on `crosses_boundary`, and the same empty Campaign passed through + `standing`, `entitlement`, `entitlement_summary` and `untrusted_instruments`, clearing + every crossing check because zero planned work crosses nothing. A planner who forgot to + fill the campaign in got a workcell-wide clean bill from three of the four doors. + """ + interval = _calibration() + health = _health(interval, since=Accrued(runs=1.0)) + wc = Workcell.default() + key = wc.present_keys()[0] + supplied = {key: _health(_calibration(instrument=key), since=Accrued(runs=1.0))} + + doors = ( + lambda: standing(health, interval, Campaign()), + lambda: health.standings(Campaign()), + lambda: health.entitlement(Campaign()), + lambda: crosses_boundary(health, Campaign()), + lambda: entitlement_summary(wc, supplied, Campaign()), + lambda: untrusted_instruments(wc, supplied, Campaign()), + # The instrument with no obligations short-circuits before its standings are built, + # so it is the one door an empty campaign could still walk through. + lambda: InstrumentHealth(instrument="star").entitlement(Campaign()), + ) + for door in doors: + with pytest.raises(ValueError, match="crosses nothing"): + door() + + assert health.entitlement() is Entitlement.ENTITLED, "no plan at all is a different question" + + +def test_a_campaign_of_negative_or_undefined_work_is_refused(): + """A campaign of minus five runs is not empty, so it survived the empty-campaign refusal. + + It also crosses nothing, ever, because it gives budget back -- and NaN clears every + boundary in the module for the same reason a NaN interval never comes due. + """ + for kwargs in ({"runs": -5.0}, {"days": -1.0}, {"hours": -0.5}): + with pytest.raises(ValueError, match="is not a plan"): + Campaign(**kwargs) + with pytest.raises(ValueError, match="is not a plan"): + Campaign(runs=float("nan")) + with pytest.raises(ValueError, match="is not a plan"): + Campaign(days=float("inf")) + + def test_expiring_is_a_deadline_rather_than_a_defect(): """EXPIRING must not read as unfit; the fix is to the plan, not to the machine.""" assert Entitlement.EXPIRING.entitled_now @@ -329,6 +481,140 @@ def test_a_nonpositive_interval_is_refused(): _calibration(every_runs=0.0) +def test_an_interval_that_recurs_on_a_number_that_never_arrives_is_refused(): + """The same never-comes-due obligation, written as a number instead of a missing field. + + The guard above refuses an interval that recurs on no unit and names the reason: an + obligation that never comes due is the vacuous pass this package hunts, arriving through + a data table. `every_days=inf` is that obligation, and `<= 0` is False for both inf and + NaN, so both walked past a guard written for exactly them. + """ + for kwargs in ( + {"every_runs": float("inf")}, + {"every_runs": float("nan")}, + {"every_days": float("inf"), "every_runs": None}, + {"every_hours": float("nan"), "every_runs": None}, + ): + with pytest.raises(ValueError, match="never comes due"): + _calibration(**kwargs) + + +def test_an_obligation_that_names_no_remedy_is_refused(): + """An author-settable field that silences half the report. + + `untrusted_instruments` collects remedies by filtering out empty strings, so a blank + `restores` produces an instrument named untrusted with nothing named that would fix it. + The repo's own guard cannot catch it today: with OBLIGATIONS empty, every row takes the + no-obligations branch whose remedy is a hardcoded literal, so the assertion is satisfied + over a path nothing exercises. + """ + with pytest.raises(ValueError, match="does not say what discharging it restores"): + _calibration(restores="") + with pytest.raises(ValueError, match="does not say what discharging it restores"): + _calibration(restores=" ") + + wc = Workcell.default() + key = wc.present_keys()[0] + interval = _calibration(instrument=key, every_runs=10.0) + rows = untrusted_instruments(wc, {key: _health(interval, since=Accrued(runs=999.0))}) + row = [r for r in rows if r.instrument == key][0] + assert row.entitlement is Entitlement.LAPSED + assert row.restores == (interval.restores,), "a lapse from a declared obligation names its fix" + + +# -- which record discharges which obligation, on which box -------------------- + + +def test_two_records_of_one_discharge_do_not_resolve_by_tuple_order(): + """The same facts must not return opposite verdicts depending on declaration order. + + `record_for` takes the last match as most recent, which is right for a service history + and wrong for two rows claiming the SAME discharge with incompatible counts. Resolved by + position, appending a flattering row was enough to bury a lapse. + """ + interval = _calibration(every_runs=100.0) + overdue = _serviced(interval, Accrued(runs=400.0)) + fresh = _serviced(interval, Accrued(runs=1.0)) + kwargs = dict(instrument=interval.instrument, obligations=(interval,)) + + first = InstrumentHealth(records=(overdue, fresh), **kwargs) + second = InstrumentHealth(records=(fresh, overdue), **kwargs) + assert first.entitlement() is second.entitlement(), "tuple order must not decide the verdict" + assert first.entitlement() is Entitlement.UNKNOWN, "two incompatible histories are unmeasured" + assert first.conflicting() and second.conflicting() + assert "disagree" in first.standings()[0].reason + assert len(first.records_for(interval.name)) == 2 + + later = ServiceRecord( + interval=interval.name, + performed="day 300", + by="technician", + attestation=Attestation.WITNESSED, + since=Accrued(runs=1.0), + ) + history = InstrumentHealth(records=(overdue, later), **kwargs) + assert history.entitlement() is Entitlement.ENTITLED, "separate discharges are a history" + assert history.conflicting() == () + assert history.record_for(interval.name) is later + + +def test_one_record_cannot_discharge_two_obligations_sharing_a_name(): + """A service record names the interval it discharged and nothing else. + + Two obligations sharing a name are therefore discharged by one signature, and a + calibration and a qualification that were never both performed both read as done. + 'annual_calibration', 'pm' and 'tip_change' are the names a real lab reuses. + """ + calibration = _calibration(name="annual", every_days=365.0, every_runs=None) + qualification = Interval( + name="annual", + instrument=calibration.instrument, + kind=IntervalKind.QUALIFICATION, + restores="requalify the pump head against the reference plate", + basis=Basis.VENDOR, + every_days=365.0, + ) + with pytest.raises(ValueError, match="declared twice"): + InstrumentHealth( + instrument=calibration.instrument, + obligations=(calibration, qualification), + records=(_serviced(calibration, Accrued(days=10.0)),), + ) + + +def test_a_health_record_filed_under_another_instrument_entitles_nothing(): + """One calibration, performed once, on one box, reported nine boxes entitled. + + `Interval.instrument` and `InstrumentHealth.instrument` are two stored copies of one + fact and nothing compared them, so the workcell view honored whatever key a record was + filed under. The docstring defends against omitting an instrument; the failure here is + misattribution, which produces a PASS rather than a gap. + """ + wc = Workcell.default() + keys = list(wc.present_keys()) + list(wc.federated) + one = _health(_calibration(every_runs=100.0), since=Accrued(runs=1.0)) + assert one.entitlement() is Entitlement.ENTITLED, "the record is real; only its filing is wrong" + + with pytest.raises(ValueError, match="filed under"): + entitlement_summary(wc, {k: one for k in keys}) + with pytest.raises(ValueError, match="filed under"): + untrusted_instruments(wc, {k: one for k in keys}) + + +def test_an_obligation_is_resolved_only_against_its_own_instruments_history(): + """One box's paperwork does not entitle another, at either level. + + Loaded through the module tables this reported tecan ENTITLED on star's pump service, + carrying a Standing whose `.instrument` and whose `.interval.instrument` disagreed. + """ + with pytest.raises(ValueError, match="does not entitle another"): + InstrumentHealth(instrument="star", obligations=(_calibration(instrument="tecan"),)) + + liquid_handler = _health(_calibration(instrument="liquid_handler", every_runs=100.0), since=Accrued(runs=1.0)) + with pytest.raises(ValueError, match="does not entitle another"): + standing(liquid_handler, _calibration(instrument="tecan")) + + # -- no field may silence the report ------------------------------------------- @@ -462,3 +748,65 @@ def test_every_instrument_lands_in_exactly_one_bucket(): buckets = sum(counts[e.value] for e in Entitlement) assert buckets == counts["total"] assert counts[Entitlement.UNKNOWN.value] == counts["total"] + + +def test_a_workcell_that_declares_no_instrument_is_not_a_clean_lab(): + """The `all()`-over-nothing shape, one level up from the obligation list. + + The natural call is `if not untrusted_instruments(wc): print("every instrument + entitled")`, and an empty list read exactly that way for a workcell that declares no + instruments at all. `BoundaryReport.clear` guards its own version of this with + `bool(self.rows)` and nothing here did. Emptiness is the finding, not the absence of one. + """ + empty = Workcell(name="empty") + assert empty.present_keys() == [] and empty.federated == () + + rows = untrusted_instruments(empty) + assert rows, "an empty list reads as a fully entitled lab" + assert rows[0].entitlement is Entitlement.UNKNOWN + assert rows[0].restores, "even this row names what would fix it" + + with pytest.raises(ValueError, match="declares no present or federated instrument"): + entitlement_summary(empty) + + +# -- the prose this module rests on --------------------------------------------- + + +def test_the_vacuous_pass_this_module_cites_is_one_qc_actually_has(): + """A precedent has to be a case the sibling really has, checked rather than asserted. + + `qc.evaluate` refuses an empty measurement dict by name, so citing that as the vacuous + pass would be a claim this repo's own code contradicts. The real analogue for an empty + obligation list is a gate that declares no criteria: `qc.readiness` refuses it and + `qc.evaluate` does not. + """ + from autonomous_lab.ledger import build_ledger + from autonomous_lab.protocols import SINGLE_CELL_GENOMICS + from autonomous_lab.qc import LIBRARY_QUANT_GATE, Decision, Gate, Readiness, evaluate, readiness + + absent = evaluate(LIBRARY_QUANT_GATE, {}) + assert not absent.ok and absent.decision is not Decision.CONTINUE + assert "not a pass" in absent.reason, "the empty measurement dict is refused by name" + + declares_nothing = Gate(name="declares_nothing", after_step="x", criteria=(), blocks="y") + assert evaluate(declares_nothing, {}).ok, "the case durability cites: no criteria, and it passes" + refused = readiness(declares_nothing, build_ledger(SINGLE_CELL_GENOMICS), {}) + assert refused.readiness is Readiness.UNSATISFIABLE, "and the sibling that refuses it" + + doc = InstrumentHealth.__doc__ or "" + assert "declares no" in doc and "criteria" in doc + assert "qc.readiness" in doc + + +def test_no_refusal_rests_on_a_claim_about_a_literature(): + """A refusal states what a number would require. It does not survey a field. + + "which no published autonomous-lab work reports" is a universal negative with no + citation, carried in a string, inside a module whose argument is that an unsourced + number becomes a specification. Nothing in this package can check it. + """ + for r in NOT_COMPUTED: + text = f"{r.why} {r.what_it_would_take}".lower() + for claim in ("no published", "nobody has published", "has never been reported", "no one has shown"): + assert claim not in text, f"'{r.quantity}' rests on a claim nothing here can check" diff --git a/tests/test_feedback.py b/tests/test_feedback.py index 6bf2474..8ee1a82 100644 --- a/tests/test_feedback.py +++ b/tests/test_feedback.py @@ -20,8 +20,13 @@ import pytest from autonomous_lab import Workcell, build_ledger, protocols -from autonomous_lab.model import Protocol, Step, Verdict, ZeroDecodeOp -from autonomous_lab.qc import GATES, PROTOCOL_GATES, Basis +from autonomous_lab.model import Artifact, Protocol, Step, Verdict, ZeroDecodeOp +from autonomous_lab.qc import ( + GATES, + MEASUREMENTS, + PROTOCOL_GATES, + Basis, +) from autonomous_lab.workcell import InstrumentConfig from autonomous_lab import feedback from autonomous_lab.feedback import ( @@ -44,7 +49,19 @@ # -- fixtures ------------------------------------------------------------------ -def _env(quantity="q", lower=0.5, upper=None, basis=Basis.IN_HOUSE): +# The quantity the fixture protocols genuinely sense. qc declares it, qc says it arrives on +# the artifact 'run_outcome', and the fixture's sensor step is the step that produces that +# artifact -- so a loop built out of these has a real sensor-to-quantity relationship rather +# than an author's say-so. A fixture sensing a name qc has never heard of, off a step that +# produces nothing, is the counterexample this module exists to refuse, and a suite whose +# definition of "well formed" was that shape could not catch it. +SENSED = "run_yield_pf" + + +def _env(quantity=SENSED, lower=0.5, upper=None, basis=Basis.IN_HOUSE): + """Units come from qc for any quantity qc declares, and are invented only where it cannot.""" + if quantity in MEASUREMENTS: + return Envelope.for_quantity(quantity, lower=lower, upper=upper, basis=basis) return Envelope(quantity=quantity, units="u", lower=lower, upper=upper, basis=basis) @@ -54,7 +71,7 @@ def _loop( envelope=None, correction=Correction.HOLD, applied_by=Controller.CODED, - quantity="q", + quantity=SENSED, name="test_loop", ): return Loop( @@ -68,48 +85,95 @@ def _loop( ) -def _two_automated_steps(): - """A protocol whose every step runs headless, so nothing but the loop can be at fault.""" +def _sensor_then_actuator(): + """Every step headless AND the sensor genuinely produces the quantity's artifact. + + Both halves matter. Headless means the ledger cannot be what refuses a loop here, and the + sensor producing 'run_outcome' means qc's own resolution of run_yield_pf lands on this + step, so nothing but the loop can be at fault. + """ + return Protocol( + name="tiny", + summary="test", + artifacts=(Artifact("run_outcome", note="run state and outcome, read off the folder"),), + steps=( + Step( + instrument="element_aviti", + op=ZeroDecodeOp.WATCH_RUN_FOLDER.value, + summary="sensor", + produces=("run_outcome",), + params={"run_dir": "/tmp/x"}, + ), + Step(instrument="element_aviti", op=ZeroDecodeOp.PROBE_HTTP.value, summary="actuator"), + ), + ) + + +def _actuator_then_sensor(): + """The same two steps with the sensor after the actuator, so only the ORDER is wrong. + + Kept apart from the well-formed fixture rather than expressed by swapping a loop's two + arguments, because the sensor has to stay the step that really produces the number for the + lateness refusal to be the only thing under test. + """ return Protocol( name="tiny", summary="test", + artifacts=(Artifact("run_outcome", note="run state and outcome, read off the folder"),), steps=( - Step(instrument="element_aviti", op=ZeroDecodeOp.PROBE_HTTP.value, summary="sensor"), + Step(instrument="element_aviti", op=ZeroDecodeOp.PROBE_HTTP.value, summary="actuator"), Step( instrument="element_aviti", op=ZeroDecodeOp.WATCH_RUN_FOLDER.value, - summary="actuator", + summary="sensor", + produces=("run_outcome",), params={"run_dir": "/tmp/x"}, ), ), ) +def _two_automated_steps(): + return _sensor_then_actuator() + + def _three_automated_steps(): return Protocol( name="tiny3", summary="test", + artifacts=(Artifact("run_outcome", note="run state and outcome, read off the folder"),), steps=( - Step(instrument="element_aviti", op=ZeroDecodeOp.PROBE_HTTP.value, summary="sensor"), - Step(instrument="namocell", op=ZeroDecodeOp.DISCOVER_USB.value, summary="in between"), Step( instrument="element_aviti", op=ZeroDecodeOp.WATCH_RUN_FOLDER.value, - summary="actuator", + summary="sensor", + produces=("run_outcome",), params={"run_dir": "/tmp/x"}, ), + Step(instrument="namocell", op=ZeroDecodeOp.DISCOVER_USB.value, summary="in between"), + Step(instrument="element_aviti", op=ZeroDecodeOp.PROBE_HTTP.value, summary="actuator"), ), ) def _blocked_sensor_protocol(): - """A sensor step the workcell cannot drive, sitting after the step it would correct.""" + """A sensor step the workcell cannot drive, sitting after the step it would correct. + + The sort produces 'sorted_plate', which is where qc says well_occupancy comes from, so + this loop's sensor is real and every one of its four refusals is earned somewhere else. + """ return Protocol( name="blocked", summary="test", + artifacts=(Artifact("sorted_plate", physical=True, note="96-well, one cell per well"),), steps=( Step(instrument="namocell", op=ZeroDecodeOp.DISCOVER_USB.value, summary="actuator"), - Step(instrument="namocell", op="start_sort", summary="sensor"), + Step( + instrument="namocell", + op="start_sort", + summary="sensor", + produces=("sorted_plate",), + ), ), ) @@ -125,13 +189,29 @@ def _costed(protocol, wc=None): def test_a_well_formed_loop_closes(): + """The positive control, and it has to be a REAL loop or it controls nothing. + + Every leg is checkable against something other than the declaration: the quantity is one + qc declares, the sensor step is the step that produces the artifact qc says the number + arrives on, both steps run headless, and a coded controller may stop a line. A positive + control built on a quantity nothing measures would pass here whether or not the module + checked the sensor at all, which is exactly how a loop over an unproduced number reports + itself closed while the suite stays green. + """ protocol, ledger = _costed(_two_automated_steps()) - loop = _loop("probe_http", "watch_run_folder", envelope=_env()) + loop = _loop("watch_run_folder", "probe_http", envelope=_env()) closure = can_close(loop, protocol, ledger) assert closure.verdict is Closable.CLOSES assert closure.closes assert closure.blockers == () + # The control is only a control while these hold: qc knows the quantity, and the step the + # loop names is the one the protocol produces its artifact at. + assert loop.quantity in MEASUREMENTS + produced_by = MEASUREMENTS[loop.quantity].produced_by + sensor = next(s for s in protocol.steps if s.op == loop.measured_at) + assert produced_by in sensor.produces + # -- refusal one: the sensor is downstream of the actuator --------------------- @@ -139,11 +219,11 @@ def test_a_well_formed_loop_closes(): def test_a_sensor_downstream_of_its_actuator_is_refused_with_that_reason(): """The load-bearing check. A measurement taken after the correction point steers nothing. - Everything else about this loop is right: it has an envelope, both steps run headless, and - a coded controller may make the correction. The only thing wrong is the order, and the - order alone must refuse it. + Everything else about this loop is right: it has an envelope, both steps run headless, a + coded controller may make the correction, and the sensor really is the step that produces + the number. The only thing wrong is the order, and the order alone must refuse it. """ - protocol, ledger = _costed(_two_automated_steps()) + protocol, ledger = _costed(_actuator_then_sensor()) loop = _loop("watch_run_folder", "probe_http", envelope=_env()) closure = can_close(loop, protocol, ledger) assert closure.verdict is Closable.OPEN_LOOP_SENSOR_TOO_LATE @@ -165,7 +245,7 @@ def test_measuring_at_the_step_being_corrected_is_open_too(): package does not model is the same overclaim as calling an unbenchmarked robot trusted. """ protocol, ledger = _costed(_two_automated_steps()) - loop = _loop("probe_http", "probe_http", envelope=_env()) + loop = _loop("watch_run_folder", "watch_run_folder", envelope=_env()) closure = can_close(loop, protocol, ledger) assert closure.verdict is Closable.OPEN_LOOP_SENSOR_TOO_LATE assert "atomic" in closure.reason @@ -177,7 +257,7 @@ def test_measuring_at_the_step_being_corrected_is_open_too(): def test_a_loop_steering_toward_nothing_is_refused_separately(): """Distinct from lateness, and fixed by an experiment rather than by a reordering.""" protocol, ledger = _costed(_two_automated_steps()) - loop = _loop("probe_http", "watch_run_folder", envelope=None) + loop = _loop("watch_run_folder", "probe_http", envelope=None) closure = can_close(loop, protocol, ledger) assert closure.verdict is Closable.NO_ENVELOPE assert closure.verdict is not Closable.OPEN_LOOP_SENSOR_TOO_LATE @@ -192,7 +272,7 @@ def test_absence_of_an_envelope_never_reads_as_a_pass(): reports every envelope-less loop as closed. """ protocol, ledger = _costed(_two_automated_steps()) - closure = can_close(_loop("probe_http", "watch_run_folder"), protocol, ledger) + closure = can_close(_loop("watch_run_folder", "probe_http"), protocol, ledger) assert not closure.closes assert Closable.NO_ENVELOPE in closure.refusals() @@ -211,7 +291,7 @@ def test_an_inverted_envelope_is_refused(): def test_an_envelope_on_a_neighbouring_quantity_is_not_an_envelope_for_this_one(): """A declaration that a summary would count and that bounds the wrong thing.""" protocol, ledger = _costed(_two_automated_steps()) - loop = _loop("probe_http", "watch_run_folder", envelope=_env(quantity="something_else")) + loop = _loop("watch_run_folder", "probe_http", envelope=_env(quantity="something_else")) closure = can_close(loop, protocol, ledger) assert closure.verdict is Closable.NO_ENVELOPE assert "something_else" in closure.reason @@ -236,7 +316,7 @@ def test_measurement_availability_is_resolved_from_the_ledger_not_asserted(): the ledger would be worse than no control layer. """ protocol = _two_automated_steps() - loop = _loop("probe_http", "watch_run_folder", envelope=_env()) + loop = _loop("watch_run_folder", "probe_http", envelope=_env()) full = build_ledger(protocol, Workcell.default()) assert can_close(loop, protocol, full).closes @@ -282,8 +362,8 @@ def test_a_broken_instrument_and_an_unwired_one_refuse_alike_and_report_differen def test_an_unpermitted_correction_is_a_fourth_and_separate_reason(): protocol, ledger = _costed(_two_automated_steps()) loop = _loop( - "probe_http", "watch_run_folder", + "probe_http", envelope=_env(), correction=Correction.DISCARD, applied_by=Controller.MODEL, @@ -326,12 +406,15 @@ def test_a_drafting_model_still_counts_as_a_model_in_the_path(): def test_the_four_refusals_stay_four_distinct_verdicts(): """Three of these are the ones the module exists to keep apart, and each has its own fix.""" protocol, ledger = _costed(_two_automated_steps()) - late = can_close(_loop("watch_run_folder", "probe_http", envelope=_env()), protocol, ledger) - targetless = can_close(_loop("probe_http", "watch_run_folder"), protocol, ledger) + late_protocol, late_ledger = _costed(_actuator_then_sensor()) + late = can_close( + _loop("watch_run_folder", "probe_http", envelope=_env()), late_protocol, late_ledger + ) + targetless = can_close(_loop("watch_run_folder", "probe_http"), protocol, ledger) unpermitted = can_close( _loop( - "probe_http", "watch_run_folder", + "probe_http", envelope=_env(), correction=Correction.DISCARD, applied_by=Controller.MODEL, @@ -358,6 +441,7 @@ def test_a_loop_broken_four_ways_reports_all_four_and_headlines_the_structural_o envelope=None, correction=Correction.DISCARD, applied_by=Controller.MODEL, + quantity="well_occupancy", ) closure = can_close(loop, protocol, ledger) assert closure.refusals() == ( @@ -436,7 +520,7 @@ def test_the_quantities_this_module_will_not_emit_are_named(): def test_in_flight_exposure_is_a_plate_count_and_not_a_duration(): """The honest form of 'how fast can this loop react', in the unit the protocol knows.""" protocol = _three_automated_steps() - flight = in_flight_exposure(_loop("probe_http", "watch_run_folder", envelope=_env()), protocol) + flight = in_flight_exposure(_loop("watch_run_folder", "probe_http", envelope=_env()), protocol) assert flight.steps_between == 1 assert flight.intervening == ("discover_usb",) assert not flight.immediate @@ -450,14 +534,14 @@ def test_in_flight_exposure_is_a_plate_count_and_not_a_duration(): def test_a_correction_landing_at_the_next_step_commits_nothing_in_between(): protocol = _two_automated_steps() - flight = in_flight_exposure(_loop("probe_http", "watch_run_folder", envelope=_env()), protocol) + flight = in_flight_exposure(_loop("watch_run_folder", "probe_http", envelope=_env()), protocol) assert flight.steps_between == 0 assert flight.immediate def test_in_flight_exposure_refuses_a_loop_that_never_reaches_its_material(): """Zero would read as the best possible case rather than as the absence of a loop.""" - protocol = _two_automated_steps() + protocol = _actuator_then_sensor() with pytest.raises(ValueError, match="measures at or after"): in_flight_exposure(_loop("watch_run_folder", "probe_http", envelope=_env()), protocol) @@ -493,7 +577,7 @@ def test_a_ledger_built_for_another_protocol_is_refused(): protocol = _two_automated_steps() other = build_ledger(protocols.get("single_cell_genomics")) with pytest.raises(ValueError, match="built for protocol"): - can_close(_loop("probe_http", "watch_run_folder", envelope=_env()), protocol, other) + can_close(_loop("watch_run_folder", "probe_http", envelope=_env()), protocol, other) # -- envelopes come from qc, never from here ----------------------------------- @@ -535,7 +619,7 @@ def test_an_unvalidated_envelope_is_reported_and_does_not_refuse_the_loop(): all. It is not: this one needs a titration, and the loop works meanwhile. """ protocol, ledger = _costed(_two_automated_steps()) - loop = _loop("probe_http", "watch_run_folder", envelope=_env(basis=Basis.VENDOR)) + loop = _loop("watch_run_folder", "probe_http", envelope=_env(basis=Basis.VENDOR)) closure = can_close(loop, protocol, ledger) assert closure.closes assert not loop.envelope.validated diff --git a/tests/test_teaching.py b/tests/test_teaching.py index 9a290a4..2d7edd3 100644 --- a/tests/test_teaching.py +++ b/tests/test_teaching.py @@ -24,14 +24,18 @@ from __future__ import annotations import dataclasses +import enum +import itertools import pytest -from autonomous_lab import protocols -from autonomous_lab.intelligence import BENCHMARKS_BY_OP +from autonomous_lab import protocols, teaching +from autonomous_lab.intelligence import BENCHMARKS_BY_OP, BenchmarkStatus from autonomous_lab.model import Protocol, Step +from autonomous_lab.qc import MEASUREMENTS, SORT_OCCUPANCY_GATE, Basis from autonomous_lab.teaching import ( DEMONSTRATIONS, + EXEMPT, MACHINE_OBSERVATIONS, MIN_DEMONSTRATIONS, TRANSFERABLE, @@ -55,7 +59,20 @@ LYSIS = "wgs_prep_lysis" # WINDOW: delivered volume -def _demo(value: float, op: str = CLEANUP, by: str = "scientist_a", conditions=None) -> Demonstration: +# Every helper-built record gets its own source. Two runs of one operation are two runs +# because they happened separately, and the evidence string is where that separateness is +# recorded; a helper that stamped them all identically would be manufacturing the duplicate +# the module now refuses, in every fixture in this file. +_RUNS = itertools.count(1) + + +def _demo( + value: float, + op: str = CLEANUP, + by: str = "scientist_a", + conditions=None, + evidence=None, +) -> Demonstration: spec = TRANSFERABLE_BY_OP[op] return Demonstration( operation=op, @@ -64,18 +81,27 @@ def _demo(value: float, op: str = CLEANUP, by: str = "scientist_a", conditions=N value=value, by=by, conditions=conditions if conditions is not None else spec.conditions, + evidence=evidence if evidence is not None else f"notebook page {next(_RUNS)}", ) -def _obs(value: float, op: str = CLEANUP, conditions=None) -> MachineObservation: +def _obs( + value: float, + op: str = CLEANUP, + conditions=None, + metric=None, + units=None, + evidence=None, +) -> MachineObservation: spec = TRANSFERABLE_BY_OP[op] return MachineObservation( operation=op, - metric=spec.metric, - units=spec.units, + metric=spec.metric if metric is None else metric, + units=spec.units if units is None else units, value=value, by="a run card", conditions=conditions if conditions is not None else spec.conditions, + evidence=evidence if evidence is not None else f"run {next(_RUNS)}", ) @@ -336,9 +362,18 @@ def test_the_top_of_the_queue_is_not_the_first_thing_declared(): def test_an_empty_queue_over_no_protocols_is_not_a_finished_programme(): - """Nothing to rank must not read as nothing left to do.""" - assert demonstration_queue([]) == [] + """Nothing to rank must not read as nothing left to do. + + Asserting the empty value is asserting the ambiguity: `== []` is exactly what a lab that + had finished teaching its machines would also return, so the queue has to REFUSE rather + than come back empty, and the refusal is what this asserts. + """ + nothing = demonstration_queue([]) + assert len(nothing) == 0 + assert nothing.operations_considered == 0 + assert nothing.refusal() is not None assert not transfer_report([]).transfers() + assert transfer_report([]).refusal() is not None assert demonstration_queue([protocols.get("single_cell_genomics")]) @@ -428,6 +463,7 @@ def test_an_envelope_refuses_to_pool_two_different_metrics(): value=85.0, by="scientist_b", conditions=spec.conditions, + evidence="notebook page 91", ) with pytest.raises(ValueError, match="two metrics pooled"): Envelope( @@ -454,7 +490,7 @@ def test_an_envelope_refuses_a_demonstration_of_another_operation(): def test_an_untaught_operation_is_named_in_its_own_refusal(): """The untaught case is the common one, and it is the case where neither the envelope nor the observations carry the operation's name -- both are absent. Without the caller - supplying it the refusal reads "no expert has demonstrated 'unknown'", which names nothing + supplying it the refusal reads "no expert envelope places 'unknown'", which names nothing and is useless as a queue entry, in the report whose entire job is to be that queue. """ report = transfer_report(["library_pool"]) @@ -466,3 +502,465 @@ def test_an_untaught_operation_is_named_in_its_own_refusal(): # And the bare function still works when the caller does not supply it. assert attainment(None, (), operation="named_op").operation == "named_op" assert attainment(None, ()).operation == "unknown" + + +# -- the machine side gets the checks the expert side already had --------------- +# Every test above this line places a well-formed observation. The observations below are +# the malformed ones, and they are the ones that arrive from a script rather than from the +# person who measured, which is why they are the ones that reach a report unchallenged. + + +def test_a_machine_observation_in_other_units_is_refused_rather_than_compared(): + """Recovery logged in percent against an envelope in fractions is not a comparison. + + The mirror of `test_an_envelope_refuses_to_pool_two_different_metrics`, on the side the + original only trusted. Untouched, 85.0 percent placed against a 0.80-0.90 fraction band + reads as MEETS and the prose prints the machine's value beside the ENVELOPE's unit. + """ + env = _envelope([0.80, 0.85, 0.90]) + percent = [_obs(85.0, units="percent"), _obs(88.0, units="percent"), _obs(90.0, units="percent")] + with pytest.raises(ValueError, match="two metrics pooled"): + attainment(env, percent) + + +def test_a_machine_observation_on_another_metric_is_refused(): + env = _envelope([0.80, 0.85, 0.90]) + renamed = [_obs(v, metric="fraction of wells with one cell") for v in (0.85, 0.88, 0.90)] + with pytest.raises(ValueError, match="two metrics pooled"): + attainment(env, renamed) + + +def test_a_machine_observation_of_another_operation_is_refused(): + """A sort occupancy filed under a cleanup must not read as that cleanup's parity.""" + spec = TRANSFERABLE_BY_OP[CLEANUP] + env = _envelope([0.80, 0.85, 0.90]) + misfiled = [ + MachineObservation( + operation="start_sort", + metric=spec.metric, + units=spec.units, + value=0.99, + by="a run card", + conditions=spec.conditions, + evidence=f"run {i}", + ) + for i in range(3) + ] + with pytest.raises(ValueError, match="two metrics pooled"): + attainment(env, misfiled) + + +def test_the_worst_run_is_reported_in_the_units_of_the_run(): + """The prose reads its unit off the observation, so a mismatch cannot hide in a sentence.""" + env = _envelope([10.0, 12.0, 14.0], op=POOLING) + parity = attainment(env, [_obs(v, op=POOLING) for v in (5.0, 6.0, 7.0)]) + assert parity.attainment is Attainment.MEETS + assert f"7.0 {TRANSFERABLE_BY_OP[POOLING].units}" in parity.reason + + +# -- an envelope is a range over one experiment --------------------------------- + + +def test_an_envelope_refuses_to_pool_two_sets_of_conditions(): + """The failure the module says it exists to prevent, on the side nothing was checking. + + Two demonstrations at 0.80 and 0.82 plus one at 0.30 under ten times the input pooled to + a 0.30-0.82 band, and three machine runs at a third of the expert's recovery then read as + MEETS against a floor the expert never produced under those conditions. + """ + spec = TRANSFERABLE_BY_OP[CLEANUP] + with pytest.raises(ValueError, match="one set of conditions"): + Envelope( + operation=CLEANUP, + metric=spec.metric, + units=spec.units, + goal=spec.goal, + demonstrations=( + _demo(0.80), + _demo(0.82), + _demo(0.30, conditions="ten times the input amount"), + ), + ) + + +def test_an_off_condition_demonstration_cannot_widen_the_band_into_a_pass(monkeypatch): + """Adding a demonstration under conditions the machine never ran at must not flip a verdict.""" + nanogram = tuple(_demo(v) for v in (0.85, 0.88, 0.90)) + picogram = (_demo(0.40, conditions="picogram spike-in of known quantity"),) + runs = tuple(_obs(v) for v in (0.45, 0.50, 0.55)) + + monkeypatch.setattr(teaching, "DEMONSTRATIONS", nanogram) + monkeypatch.setattr(teaching, "MACHINE_OBSERVATIONS", runs) + monkeypatch.setattr(teaching, "ENVELOPES", teaching._build_envelopes()) + control = teaching.transfer_report([CLEANUP]) + assert control.counts()["below"] == 1 + + monkeypatch.setattr(teaching, "DEMONSTRATIONS", nanogram + picogram) + monkeypatch.setattr(teaching, "ENVELOPES", teaching._build_envelopes()) + widened = teaching.transfer_report([CLEANUP]) + assert widened.counts()["below"] == 1, "an off-condition demonstration widened the band" + assert not widened.transfers() + # The picogram run is its own experiment, and it keeps its own envelope. + assert len(teaching.envelopes_for(CLEANUP)) == 2 + assert teaching.envelope_for(CLEANUP) is None + assert "different sets of conditions" in taught(CLEANUP)[1] + + +# -- goal comes from the spec, and the comparison reads it off the enum --------- + + +def test_an_envelope_cannot_redeclare_which_direction_is_better(): + """The one field that flips a verdict, checked against the layer that owns it. + + `test_no_field_anywhere_declares_a_tolerance_or_an_attainment` enumerates `goal` as an + acceptable field name. It is acceptable only because a mismatched one is refused, and a + test on names alone would bless the field that turns BELOW into MEETS. + """ + for op, spec in TRANSFERABLE_BY_OP.items(): + wrong = Goal.LOWER if spec.goal is not Goal.LOWER else Goal.HIGHER + with pytest.raises(ValueError, match="is better"): + Envelope(operation=op, metric=spec.metric, units=spec.units, goal=wrong) + + +def test_an_envelope_cannot_restate_the_metric_or_the_units_either(): + spec = TRANSFERABLE_BY_OP[CLEANUP] + with pytest.raises(ValueError, match="is better"): + Envelope(operation=CLEANUP, metric=spec.metric, units="percent", goal=spec.goal) + + +def test_an_envelope_for_an_operation_nobody_specified_is_refused(): + """`_build_envelopes` already treats this as an error; the public constructor must too.""" + with pytest.raises(ValueError, match="no Transferable declares"): + Envelope( + operation="an_operation_nobody_ever_wrote_down", + metric="something", + units="units", + goal=Goal.HIGHER, + ) + + +def test_leaving_the_range_the_better_way_is_a_failure_exactly_where_the_goal_says_so(): + """The two-sided rule is read off `Goal.outside_is_always_worse`, not restated as an else. + + Every Goal member is exercised, so a fourth one cannot inherit WINDOW semantics from a + fall-through while the property reports False for it. + """ + cases = ( + (CLEANUP, [0.80, 0.85, 0.90], (0.95, 0.96, 0.97)), # HIGHER: above the band + (POOLING, [10.0, 12.0, 14.0], (5.0, 6.0, 7.0)), # LOWER: below the band + (LYSIS, [9.8, 10.0, 10.2], (11.0, 11.1, 11.2)), # WINDOW: above the band + ) + seen = set() + for op, demonstrated, runs in cases: + goal = TRANSFERABLE_BY_OP[op].goal + seen.add(goal) + env = _envelope(demonstrated, op=op) + parity = attainment(env, [_obs(v, op=op) for v in runs]) + beyond_is_a_failure = parity.attainment is Attainment.BELOW + assert beyond_is_a_failure is goal.outside_is_always_worse, op + assert seen == set(Goal), "a Goal member no test reaches is a branch nobody checks" + + +def test_a_goal_the_comparison_does_not_handle_is_refused_rather_than_defaulted(monkeypatch): + """A fourth direction must reach a refusal, not the two-sided branch by accident.""" + + class _Centered(str, enum.Enum): + CENTERED = "centered" + + @property + def outside_is_always_worse(self) -> bool: + return False + + spec = TRANSFERABLE_BY_OP[CLEANUP] + monkeypatch.setitem( + teaching.TRANSFERABLE_BY_OP, CLEANUP, dataclasses.replace(spec, goal=_Centered.CENTERED) + ) + env = Envelope( + operation=CLEANUP, + metric=spec.metric, + units=spec.units, + goal=_Centered.CENTERED, + demonstrations=tuple(_demo(v) for v in (0.80, 0.85, 0.90)), + ) + with pytest.raises(ValueError, match="does not handle"): + attainment(env, [_obs(v) for v in (0.85, 0.86, 0.87)]) + + +# -- a verdict against the machine has to be readable --------------------------- + + +def test_a_machine_below_the_expert_is_readable_in_the_report(monkeypatch): + """The only verdict that says the machine performs worse, and it had no accessor at all.""" + monkeypatch.setattr(teaching, "DEMONSTRATIONS", tuple(_demo(v) for v in (0.80, 0.85, 0.90))) + monkeypatch.setattr(teaching, "MACHINE_OBSERVATIONS", tuple(_obs(v) for v in (0.10, 0.12, 0.11))) + monkeypatch.setattr(teaching, "ENVELOPES", teaching._build_envelopes()) + report = teaching.transfer_report([CLEANUP]) + assert [r.operation for r in report.below()] == [CLEANUP] + assert report.counts()["below"] == 1 + assert not report.transfers() + + +def test_relabelling_the_conditions_cannot_erase_a_verdict_without_trace(monkeypatch): + """`conditions` is free text on both sides, so the erasure has to leave a count behind.""" + monkeypatch.setattr(teaching, "DEMONSTRATIONS", tuple(_demo(v) for v in (0.80, 0.85, 0.90))) + monkeypatch.setattr(teaching, "ENVELOPES", teaching._build_envelopes()) + + monkeypatch.setattr(teaching, "MACHINE_OBSERVATIONS", tuple(_obs(v) for v in (0.10, 0.12, 0.11))) + stated = teaching.transfer_report([CLEANUP]).counts() + + elsewhere = tuple(_obs(v, conditions="a condition nobody demonstrated") for v in (0.10, 0.12, 0.11)) + monkeypatch.setattr(teaching, "MACHINE_OBSERVATIONS", elsewhere) + silenced = teaching.transfer_report([CLEANUP]) + assert silenced.counts() != stated + assert silenced.counts()["discarded_observations"] == 3 + assert silenced.rows[0].parity.machine_n == 3 + assert silenced.rows[0].parity.matched_n == 0 + + +def test_every_row_lands_in_exactly_one_verdict_bucket(): + """Buckets that do not sum are buckets with somewhere for a verdict to go missing.""" + protocol = protocols.get("single_cell_genomics") + counts = transfer_report([s.op for s in protocol.steps]).counts() + assert ( + counts["attained"] + counts["below"] + counts["unmeasured"] + counts["unplaceable"] + == counts["operations"] + ) + + +def test_a_met_benchmark_is_not_a_target_nobody_has_hit(): + """`asserted_only` resolves the STATUS, which is the property `intelligence` draws it with.""" + for op in ("tray_cycle", "iswap_to_hhs"): + row = transfer_report([op]).rows[0] + assert row.benchmarks and all(b.status is BenchmarkStatus.MET for b in row.benchmarks) + assert not row.asserted_only + assert row.met_without_envelope + unmet = transfer_report(["read_absorbance"]).rows[0] + assert unmet.benchmarks[0].status is BenchmarkStatus.UNMET + assert unmet.asserted_only + + +# -- the backlog names what nobody has specified -------------------------------- + + +def test_a_protocol_of_operations_nobody_specified_still_produces_a_backlog(): + """The most untaught operations there are, and the queue used to omit them entirely.""" + nameless = Protocol( + name="unspecified", + summary="two operations nobody ever wrote a spec for", + steps=( + Step(instrument="star", op="an_operation_nobody_ever_wrote_down", summary=""), + Step(instrument="star", op="another_undeclared_op", summary=""), + ), + ) + queue = demonstration_queue([nameless]) + assert {u.operation for u in queue.unspecified} == { + "an_operation_nobody_ever_wrote_down", + "another_undeclared_op", + } + assert all(u.demonstrations_needed == MIN_DEMONSTRATIONS for u in queue.unspecified) + assert queue.operations_considered == 2 + assert queue.refusal() is None + assert queue.cost() == 2 * MIN_DEMONSTRATIONS + # The module used to disagree with itself on this exact input. + assert {op for op, _why in untaught_operations(nameless)} == { + u.operation for u in queue.unspecified + } + + +def test_the_queue_accounts_for_every_operation_the_protocol_runs(): + """Ranked, unspecified, or exempt. Nothing is dropped on the way out of the loop.""" + protocol = protocols.get("single_cell_genomics") + queue = demonstration_queue([protocol]) + named = ( + {op for entry in queue for op in entry.operations} + | {u.operation for u in queue.unspecified} + | set(queue.exempt) + ) + assert named == {s.op for s in protocol.steps} + assert queue.operations_considered == len(named) + assert queue.unspecified, "a real protocol with nothing unspecified would be suspicious" + + +def test_an_exemption_has_to_say_why_and_cannot_double_as_a_specification(): + """An exemption removes work from a backlog, so it is the shape of a silencer.""" + assert EXEMPT + for op, why in EXEMPT.items(): + assert why.strip(), f"'{op}' is exempt and says nothing about why" + assert op not in TRANSFERABLE_BY_OP + ok, reason = taught(op) + assert not ok, f"'{op}' reads as taught because it was exempted" + assert "exempt" in reason + + +def test_a_queue_over_a_protocol_with_no_steps_is_not_a_queue_over_no_protocols(): + """Three states that used to return the same empty list.""" + stepless = Protocol(name="stepless", summary="", steps=()) + nothing_handed = demonstration_queue([]) + nothing_in_it = demonstration_queue([stepless]) + assert nothing_handed.refusal() is not None + assert nothing_in_it.refusal() is not None + assert untaught_operations(stepless).refusal() is not None + assert untaught_operations(protocols.get("single_cell_genomics")).refusal() is None + + +# -- three entries is not three measurements ------------------------------------ + + +def test_one_measurement_recorded_three_times_is_not_three_measurements(): + """The repo's own last-audit precedent: a duplicate entry defeating a gate.""" + once = _demo(0.85) + spec = TRANSFERABLE_BY_OP[CLEANUP] + with pytest.raises(ValueError, match="not two measurements"): + Envelope( + operation=CLEANUP, + metric=spec.metric, + units=spec.units, + goal=spec.goal, + demonstrations=(once, once, once), + ) + + +def test_one_machine_run_counted_three_times_is_not_three_runs(): + """The sub-case at the API level: one run repeated cleared a minimum one run cannot.""" + env = _envelope([0.80, 0.85, 0.90]) + once = _obs(0.88) + assert attainment(env, [once]).attainment is Attainment.INDISTINGUISHABLE_FROM_UNMEASURED + with pytest.raises(ValueError, match="one run counted"): + attainment(env, [once, once, once]) + + +def test_a_range_of_width_zero_over_three_runs_is_refused(): + """Three independent runs of a continuous quantity do not agree to full precision.""" + env = _envelope([0.85, 0.85, 0.85]) + assert env.n() == MIN_DEMONSTRATIONS + assert env.tolerance() is None + assert env.spread() is None + assert "transcribed" in (env.refusal() or "") + + +def test_two_numbers_copied_do_not_certify_a_transfer(monkeypatch): + """The end-to-end version. This is the test that catches the whole family.""" + monkeypatch.setattr( + teaching, "DEMONSTRATIONS", tuple(_demo(0.85, evidence=f"page {i}") for i in range(3)) + ) + monkeypatch.setattr( + teaching, "MACHINE_OBSERVATIONS", tuple(_obs(0.88, evidence=f"run {i}") for i in range(3)) + ) + monkeypatch.setattr(teaching, "ENVELOPES", teaching._build_envelopes()) + report = teaching.transfer_report([CLEANUP]) + assert not report.transfers() + assert report.counts()["attained"] == 0 + assert report.counts()["with_envelope"] == 0 + assert not taught(CLEANUP)[0] + assert teaching.demonstrations_still_needed(CLEANUP) >= 1 + + +def test_the_backlog_never_says_zero_while_the_verdict_says_untaught(): + """Two numbers a reader puts side by side, and no reader could resolve a disagreement.""" + for op in list(TRANSFERABLE_BY_OP) + ["an_operation_nobody_ever_wrote_down"]: + if not taught(op)[0]: + assert demonstrations_still_needed(op) >= 1, op + + +# -- a number with no source, and a number that is not a number ----------------- + + +def test_a_demonstration_with_no_evidence_is_refused(): + """Every sibling module carries a basis or an evidence string. This one has to as well.""" + spec = TRANSFERABLE_BY_OP[CLEANUP] + with pytest.raises(ValueError, match="records no evidence"): + Demonstration( + operation=CLEANUP, + metric=spec.metric, + units=spec.units, + value=0.85, + by="scientist_a", + conditions=spec.conditions, + evidence=" ", + ) + + +def test_a_machine_observation_with_no_evidence_is_refused(): + spec = TRANSFERABLE_BY_OP[CLEANUP] + with pytest.raises(ValueError, match="records no evidence"): + MachineObservation( + operation=CLEANUP, + metric=spec.metric, + units=spec.units, + value=0.88, + by="a run card", + conditions=spec.conditions, + evidence="", + ) + + +def test_a_run_whose_outcome_could_not_be_measured_is_not_an_observation(): + """A NaN pads n while being invisible to min and max, and min/max short-circuit on it. + + Untouched, two real demonstrations plus one non-measurement produced the tolerance two + real demonstrations alone are refused, and the machine verdict depended on input order. + """ + for bad in (float("nan"), float("inf"), float("-inf")): + with pytest.raises(ValueError, match="not an observation of the operation"): + _demo(bad) + with pytest.raises(ValueError, match="not an observation of the operation"): + _obs(bad) + + +def test_every_specified_metric_says_where_the_choice_of_metric_came_from(): + """A metric choice is a scientific claim, and this package attributes those everywhere.""" + for spec in TRANSFERABLE: + assert isinstance(spec.basis, Basis) + assert not spec.basis.validated, ( + f"'{spec.op}' claims a validated basis for its metric; no titration in this repo " + "established that this is the quantity that decides the operation" + ) + + +# -- a demonstrator handle is an identifier, not free text ---------------------- + + +def test_a_trailing_space_does_not_turn_one_person_into_three(): + """The caveat that must travel with every parity claim, defeated by whitespace.""" + env = _envelope([0.80, 0.85, 0.90], by=("scientist_a", "Scientist_A", "scientist_a ")) + assert env.demonstrators() == ("scientist_a",) + assert "one person's performance" in (env.caveat() or "") + parity = attainment(env, [_obs(v) for v in (0.88, 0.87, 0.91)]) + assert parity.attainment is Attainment.MEETS + assert "Caveat" in parity.reason + assert taught(CLEANUP)[0] is False + + +def test_a_demonstration_with_no_handle_at_all_is_refused(): + with pytest.raises(ValueError, match="records no handle"): + _demo(0.85, by=" ") + + +# -- qc owns the quantities it already names ------------------------------------ + + +def test_the_sort_metric_is_resolved_from_qc_rather_than_restated(): + """Two statements of one quantity drift the first time either is edited.""" + spec = TRANSFERABLE_BY_OP["start_sort"] + known = MEASUREMENTS[spec.measurement] + assert spec.measurement == "well_occupancy" + assert spec.metric == known.note + assert spec.units == known.units + + +def test_a_specification_cannot_restate_a_qc_measurement_differently(): + spec = TRANSFERABLE_BY_OP["start_sort"] + with pytest.raises(ValueError, match="qc owns this quantity"): + dataclasses.replace(spec, units="percent") + with pytest.raises(ValueError, match="qc does not define"): + dataclasses.replace(spec, measurement="a_measurement_qc_never_heard_of") + + +def test_the_threshold_a_gate_will_enforce_travels_beside_the_benchmark(): + """The asserted number with a value on it, and the row used to carry only the one without.""" + row = transfer_report(["start_sort"]).rows[0] + enforced = {c.name for c in row.criteria} + assert "enough_wells_occupied" in enforced + assert row.criteria[0] is SORT_OCCUPANCY_GATE.criteria[0] + assert row.benchmarks # and intelligence's target, which states no number, is still here + assert row.asserted_only