From 5cf5c5a0470c8d47e4c437ac39f6d07ab3b7fe34 Mon Sep 17 00:00:00 2001 From: di-omics Date: Tue, 28 Jul 2026 02:14:03 -0700 Subject: [PATCH 1/3] Check the ambition against the published record, not just against this lab Every layer here asks what this lab can prove. None could ask whether the plan is supported by anything published, who is allowed to decide, or what a stated throughput target actually costs. Those are the three questions that decide whether an autonomy programme is engineering or aspiration, and all three are answerable from public evidence. literature encodes fifteen works, each carrying what it does NOT establish, because autonomous-lab claims are routinely justified by pointing at famous papers whose scope is far narrower than the claim -- a materials campaign cited to justify a cell-assay production line. Support resolves by scope rather than by fame. Every citation was independently verified against the live record before it was encoded, and the verification changed the content rather than confirming it. The A-Lab paper was corrected: 41 successes became 36 after four were reclassified as inconclusive, and the "43 new materials" figure that appears in news coverage matches no version of the paper. Assay Guidance Manual chapters carry no DOI, so any citation supplying one is fabricated. PyLabRobot makes an architectural assertion about cross-family simulation rather than demonstrating it. Thirteen entries are confirmed, two partial, and eleven carry figures that could not be confirmed -- listed rather than rounded away. authority models which decisions a model may make. "AI-native" gets read as "the model drives the instruments", and the benchmark evidence is that no evaluated model exceeded 70% hazard-identification accuracy. The honest response is not that models are useless but that decision classes differ, so thirteen classes sit across four levels and each states why it sits where it does against a specific published finding. Authority may be strengthened and never weakened; the reverse is what violations() catches. The qualification ladder is included because no rung implies the one above it, and a passed simulation qualifies nothing physical. cadence refuses a target until it says whether it counts plates started, completed, QC-passed, or released, because those four differ by the failure and rerun rate and resolving the ambiguity silently would pick the loosest reading. Given an endpoint it is arithmetic on the caller's inputs and needs no measured durations. It refuses a buffer depth for the same reason throughput refuses a makespan. demonstrated_support computes the absence of sustained-rate evidence from the table rather than asserting it, so it flips the day somebody publishes a counterexample. 131 new tests, 277 total. --- README.md | 88 +- autonomous_lab/__init__.py | 72 ++ autonomous_lab/authority.py | 842 +++++++++++++++++++ autonomous_lab/cadence.py | 1041 +++++++++++++++++++++++ autonomous_lab/cli.py | 134 +++ autonomous_lab/literature.py | 1495 ++++++++++++++++++++++++++++++++++ tests/test_authority.py | 435 ++++++++++ tests/test_cadence.py | 490 +++++++++++ tests/test_literature.py | 437 ++++++++++ 9 files changed, 5033 insertions(+), 1 deletion(-) create mode 100644 autonomous_lab/authority.py create mode 100644 autonomous_lab/cadence.py create mode 100644 autonomous_lab/literature.py create mode 100644 tests/test_authority.py create mode 100644 tests/test_cadence.py create mode 100644 tests/test_literature.py diff --git a/README.md b/README.md index c68eac8..5b5c4dc 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,10 @@ autonomous-lab throughput single_cell_genomics # plates/day, or why that number autonomous-lab provenance single_cell_genomics # what could be proven about a run afterwards autonomous-lab lineage single_cell_genomics # which cell did this read come from? autonomous-lab knowledge # encoded expert judgment and robot benchmarks + +autonomous-lab evidence # what the published record shows, and what it does not +autonomous-lab authority # which decisions a model may make, and which need a person +autonomous-lab cadence 500 --endpoint released --hours 16 # what that target actually costs ``` ## What it reports today @@ -318,6 +322,88 @@ adequacy. The STAR's dry motion is validated; its volumetric accuracy at the wor has never been measured, and no camera retires that benchmark. It needs a calibration experiment. +## Is the ambition supported by anything published? + +Every layer above asks what *this* lab can prove. Three more ask a question none of them +can, because it is not about this lab: **is the plan supported by anything published, who +is allowed to decide, and what does a stated throughput target actually cost?** + +``` +autonomous-lab evidence # what the record shows -- and what it does not +autonomous-lab authority # which decisions a model may make +autonomous-lab cadence 500 --endpoint released --hours 16 +``` + +### Evidence, scoped rather than cited + +Autonomous-lab claims get justified by pointing at famous papers whose scope is far +narrower than the claim. A materials-synthesis campaign gets cited to justify a cell-assay +production line. So every entry carries what it **does not establish**, and support is +resolved by scope rather than by fame. + +Fifteen works, each independently verified against the live record before it was encoded -- +**thirteen confirmed, two partial, and eleven carrying figures that could not be +confirmed**, listed rather than rounded away. The verification changed the content: + +- The A-Lab paper was **corrected**. Forty-one successes became thirty-six after four were + reclassified as inconclusive, and the "43 new materials" figure in news coverage matches + no version of the paper. +- Assay Guidance Manual chapters carry **no DOI**, so any citation supplying one is wrong. +- PyLabRobot makes an *architectural assertion* about cross-family simulation. It does not + demonstrate it, and the entry says so. + +`unsupported_claims()` is the standing list of things **no** cited work establishes. It is +computed from the table, so it shrinks on its own the day real evidence arrives. + +### Authority: which decisions a model may make + +"AI-native" gets read as "the model drives the instruments". The benchmark evidence is that +**no evaluated model exceeded 70% hazard-identification accuracy**, and fluent output is +not evidence of laboratory safety competence. But the honest position is not that models +are useless -- it is that decision classes differ. + +Thirteen classes across four levels: `MODEL` may decide alone, `MODEL_PROPOSES` needs a +deterministic check or a human to ratify, `DETERMINISTIC` needs a coded interlock with no +model in the path, and `HUMAN` needs a named accountable person. **Authority may be +strengthened and never weakened** -- putting a person on a decision a model could make is a +cost, and the reverse is the defect `violations()` exists to catch. + +Each class states *why* it sits where it does, against a specific published finding. Triage +sits at `MODEL_PROPOSES` because of a documented case where an unexpected intramolecular +cyclization carried the same molecular weight as the expected product, was indistinguishable +by chromatogram or MS, and passed the automated rule -- until a human read the NMR. + +The qualification ladder runs simulation -> device acceptance -> integrated dry run -> +volumetric qualification -> assay transfer -> sustained production. **No rung implies the +one above it.** A passed simulation qualifies nothing physical. + +### Cadence: what a throughput target actually costs + +A target is the easiest number in a lab to state and the easiest to state meaninglessly. +`cadence` refuses one until it says what it counts, because plates *started*, *completed*, +*QC-passed*, and *released* differ by exactly the failure and rerun rate -- a measurement, +not an assumption. Resolving that silently would pick the loosest reading in the caller's +favour. + +Given an endpoint it is pure arithmetic on the caller's inputs, needing no measured +durations: 500 plates a day is one every **172.8 s** round the clock, or **115.2 s** inside +a sixteen-hour window. The ratio between those is the walk-away penalty, and it depends on +the clock alone -- not on the target, protocol, or instruments, which is the whole +quantitative case for closing the steps that need a human. + +It then names what the interval forces, and refuses where the input does not exist: a +buffer depth is a makespan divided by an interval, so it inherits every unmeasured second +and is reported as not computable rather than estimated. `headroom()` is where "just buy a +faster robot" dies -- an intervention rate of one per two plates at ten minutes each +consumes **260%** of a 115.2 s interval, and instrument speed is not in that arithmetic. + +And `demonstrated_support()` answers the question everybody skips. No cited system +establishes integrated multi-domain operation at a production rate: the longest integrated +campaign in the record is seventeen days in a single domain, and the one system running +continuously at scale across institutions is a cage sensor that observes rather than +actuates. That emptiness is **computed from the evidence table**, so it flips the day +somebody publishes a counterexample rather than staying a stale assertion. + ## The RE queue is computed, not argued about ``` @@ -472,7 +558,7 @@ event receiver and silently steals the first one's callbacks. pip install -e '.[dev]' && pytest ``` -149 device-free tests. The ones that matter most try to make the layer lie: +277 device-free tests. The ones that matter most try to make the layer lie: - claim a step is automated when its command is undecoded; claim a decoded command is runnable while its siblings are not; claim a federated leg runs when no run card was ever diff --git a/autonomous_lab/__init__.py b/autonomous_lab/__init__.py index f6a45a9..8742922 100644 --- a/autonomous_lab/__init__.py +++ b/autonomous_lab/__init__.py @@ -35,6 +35,20 @@ lineage which cell a read came from -- and whether pooling already destroyed the answer, which is a different question from whether the plate was tracked. intelligence the tacit expert judgment and the benchmarks a robot must meet first. + +Three further layers ask the question the others cannot, because it is not about this lab +at all: is the ambition supported by anything published, who is allowed to decide, and what +does a stated throughput target actually cost? + + literature what the published record demonstrates -- and, entry by entry, what it + explicitly does NOT establish. Scope, not fame, decides whether a paper + supports a claim. + authority which decisions a model may make, which need a deterministic interlock, and + which need a named accountable human. Authority may be strengthened, never + weakened. + cadence a plates-per-day target converted into the admission interval it implies, + and refused until it says whether it counts plates started, completed, + QC-passed, or released. """ from .executor import Executor, Handoff, RunReport, StepResult @@ -48,7 +62,39 @@ loop_closure, trusted_for, ) +from .authority import ( + Authority, + DecisionClass, + Plan, + QualificationRung, + Rung, + Violation, + qualified_for, + violations, +) +from .cadence import ( + Cadence, + Demonstration, + Endpoint, + Target, + cadence_for, + demonstrated_support, + headroom, + launch_interval, + sustained_rate_demonstrations, +) from .ledger import Ledger, StepVerdict, Unlock, build_ledger, cost_step, rank_unlocks +from .literature import ( + EVIDENCE, + Confidence, + Domain, + Evidence, + EvidenceKind, + Scope, + Support, + support_for, + unsupported_claims, +) from .lineage import ( MISASSIGNMENT, Cohort, @@ -92,13 +138,23 @@ def loop_closure_for(protocol, workcell=None): __all__ = [ "Artifact", + "Authority", + "Cadence", "Cohort", + "Confidence", "MISASSIGNMENT", "Attestation", "Basis", "Benchmark", "BenchmarkStatus", "Criterion", + "DecisionClass", + "Demonstration", + "Domain", + "EVIDENCE", + "Endpoint", + "Evidence", + "EvidenceKind", "CustodyGap", "Decision", "Detection", @@ -122,23 +178,30 @@ def loop_closure_for(protocol, workcell=None): "Misassignment", "LoopClosure", "Observable", + "Plan", "Protocol", + "QualificationRung", "Readiness", "Role", "RunRecord", + "Rung", + "Scope", "Separability", "RunReport", "Severity", "Step", "StepResult", + "Support", "StepVerdict", "Tier", + "Target", "TimeBasis", "Traceability", "Transform", "UndeclaredTransform", "Unlock", "Verdict", + "Violation", "VisionCapability", "VisionRequirement", "VisualCheck", @@ -146,20 +209,29 @@ def loop_closure_for(protocol, workcell=None): "ZeroDecodeOp", "build_ledger", "build_lineage", + "cadence_for", "cost_step", "declared", + "demonstrated_support", "estimate", "evaluate", "gate_report", + "headroom", "knowledge_summary", + "launch_interval", "lineage_report", "loop_closure", "loop_closure_for", "provenance_report", + "qualified_for", "rank_unlocks", "recovery_report", "registry", "spec", + "support_for", + "sustained_rate_demonstrations", "trusted_for", "undeclared_transforms", + "unsupported_claims", + "violations", ] diff --git a/autonomous_lab/authority.py b/autonomous_lab/authority.py new file mode 100644 index 0000000..1024c39 --- /dev/null +++ b/autonomous_lab/authority.py @@ -0,0 +1,842 @@ +"""Which decisions a model may make alone, which it may only draft, and which it must not touch. + +This is the single most important question in autonomous-lab engineering and the one the +phrase "AI-native lab" is routinely used to skip. Read literally, "AI-native" says the model +drives the instruments. Every other module in this package has been arguing against that +reading from a different direction -- qc says a gate whose input never arrives is not a +gate, vision says no camera resolves a denatured enzyme, intelligence says an unmeasured +operation is not a trusted one -- and none of them says the thing directly. This one does: +authority is a property of the DECISION, not of the system that happens to be holding it. + +The evidence people usually reach for is a demo. The evidence that bears is the benchmark. +On LabSafety Bench (Zhou et al., Nature Machine Intelligence 2026), 19 evaluated models -- +8 proprietary, 7 open-weight LLMs, 4 open-weight VLMs -- none exceeded 70 percent accuracy +on the open-ended hazards identification test, while the same generation of models scored in +the mid-80s on the benchmark's multiple-choice portion (GPT-4o 86.55 percent). The gap is +the finding, not either number. Structured recall does not carry into open-ended reasoning +about a scenario, and a scenario is the form in which a real hazard arrives. Be precise +about what that does NOT establish: it is constructed text questions across BSL 1-3 aligned +to US OSHA protocols, evaluated on a late-2024 model snapshot; it measures no bench +behavior, no physical task execution, and no incident rate, and it does not show that any +model has caused a laboratory incident. It is enough for exactly one inference, and that +inference is enough: FLUENT OUTPUT IS NOT EVIDENCE OF HAZARD COMPETENCE, so fluency cannot +be the thing that authorizes an action. + +The honest position is not that models are useless. It is that decision classes differ and +each has a different required authority: + + MODEL a model may decide alone. The output commits nothing, and the error is + visible to whoever reads it next. + MODEL_PROPOSES a model drafts; a deterministic check or a human ratifies before it + becomes an action. The draft is genuinely valuable here. + DETERMINISTIC a coded interlock decides, with no model anywhere in the path. Not + because code is smarter, but because code cannot be argued into a + different answer by a well-formed account of the situation. + HUMAN a named person approves and is accountable for having approved. + +DETERMINISTIC sits ABOVE MODEL_PROPOSES on purpose, and it is the ordering people find +surprising. A drafting model is still in the path: the ratifier reads a proposal that has +already framed the problem, and a plausible frame is the thing a model produces most +reliably when it is wrong. An interlock reading a door sensor cannot be framed. Strength +here means "how little a model's output can move", not "how much intelligence is applied". + +The load-bearing computation is `violations(plan)`. A plan assigns an actual authority to +each decision class; a violation is every place the assignment is WEAKER than the class +requires. Assigning MODEL to a DETERMINISTIC or HUMAN class is the failure this module +exists to catch, and it is worth being blunt that this is not a hypothetical failure mode +but the default one, because the weakest assignment is also the fastest to build and the +best to demonstrate. + +Authority may always be STRENGTHENED and never weakened. A lab may put a human on a +decision a model could make; the reverse is the defect. `strengthened()` reports where a lab +went above the floor and nothing here ever flags it -- though over-escalation has its own +real cost, which is a human approving so many things that approval stops meaning anything, +and this module measures none of that. RoboCulture is the published example of a deliberate +strengthening: the system detected culture saturation and tracked growth on its own, and the +split was still manually initiated as a safety precaution. + +And absence of an assignment is not compliance. A plan that says nothing about batch release +has not permitted a human to decide it; it has left the question to whatever code path +reaches it first. This is the same rule qc.evaluate applies to a missing measurement and +intelligence.trusted_for applies to a missing benchmark, and it is the rule that a plan +document naturally breaks, because the classes nobody thought about are exactly the ones +nobody wrote down. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Dict, List, Optional, Sequence, Tuple + + +class Authority(str, Enum): + """Who may decide. Ordered weakest to strongest by how little a model's output can move. + + The ordering is what makes the check computable, so it has to mean one thing. It does not + rank competence, cost, or speed. It ranks the distance between a generated token and a + consequence. + """ + + MODEL = "model" # a model decides; its output is the outcome + MODEL_PROPOSES = "model_proposes" # a model drafts; a coded check or a person ratifies + DETERMINISTIC = "deterministic" # coded interlock decides; no model in the path + HUMAN = "human" # a named person approves and is accountable + + @property + def model_decides_alone(self) -> bool: + """True only for MODEL. This is the line the module exists to draw. + + Every safety argument in this package turns on whether generated output becomes an + outcome with nothing in between. At MODEL_PROPOSES a wrong draft meets a ratifier; at + DETERMINISTIC and HUMAN the model is not the decider at all. At MODEL there is nothing + downstream, which is acceptable only for classes where nothing downstream is needed. + """ + return self is Authority.MODEL + + @property + def model_in_path(self) -> bool: + """True where model output reaches the decision at all, whether deciding or drafting. + + Separate from `model_decides_alone` because the hazard classes need the stronger claim. + A hazard call does not become safe by adding a reviewer to a model's framing of it. + """ + return self in (Authority.MODEL, Authority.MODEL_PROPOSES) + + @property + def rank(self) -> int: + return _ORDER.index(self) + + def satisfies(self, required: "Authority") -> bool: + """Does an assignment at this authority meet a class that requires `required`? + + Strengthening satisfies. Weakening never does, at any distance: MODEL_PROPOSES against + a DETERMINISTIC requirement is one step short and is still a model in the path of an + interlock, which is the whole thing the requirement was written to exclude. + """ + return self.rank >= required.rank + + +# Weakest first. Used by rank, satisfies, and every report that orders by strength. +_ORDER: Tuple[Authority, ...] = ( + Authority.MODEL, + Authority.MODEL_PROPOSES, + Authority.DETERMINISTIC, + Authority.HUMAN, +) + + +@dataclass(frozen=True) +class DecisionClass: + """One kind of decision, and the weakest authority that may hold it. + + `because` is the mechanism -- why this class sits where it does -- and it is required + rather than optional. A required authority with no stated mechanism cannot be argued + with, only obeyed, and a rule that cannot be argued with cannot be revised when the + evidence changes. That is the same discipline qc.Criterion applies to a threshold and + intelligence.Judgment applies to a bench rule. + + `evidence_key` names an entry in this repo's verified-literature index. It is a string + rather than an import so that the bibliography can grow without this module changing, and + so that a class with no published evidence bearing on it says so by leaving it None + instead of borrowing a citation that does not fit. + """ + + name: str + decides: str # what is being decided, in the terms a bench would use + required: Authority + because: str + evidence_key: Optional[str] = None + + def __post_init__(self) -> None: + if not self.because.strip(): + raise ValueError( + f"decision class '{self.name}' states no mechanism; a required authority with no " + f"reason behind it cannot be revised, only obeyed" + ) + + def permits(self, assigned: Authority) -> bool: + return assigned.satisfies(self.required) + + @property + def admits_a_model(self) -> bool: + """Whether any model output may reach this decision, in any role.""" + return self.required.model_in_path + + +# -- the classes a real lab actually has to place -------------------------------- +# Ordered weakest requirement first, because that ordering is the argument: the top of this +# list is the work models are genuinely good at and should be doing, and the bottom is the +# work that a demo quietly hands them. + +DECISION_CLASSES: Tuple[DecisionClass, ...] = ( + DecisionClass( + name="prior_result_retrieval", + decides="which prior results bear on a question, and what they say", + required=Authority.MODEL, + because=( + "the output is text, nothing physical is committed by it, and the source travels with " + "the claim, so an error is visible to the next reader and costs a re-read. That is why " + "this class sits at MODEL -- not because summarization is reliable. CPJUMP1 is the " + "standing counterexample: its authors' own conclusion is that matching chemical to " + "genetic perturbations is a challenging task, so a fluent summary reporting it as " + "evidence that morphological profiling links compounds to their gene targets inverts " + "the finding it cites. A summary delivered without its source is a different class, " + "because nothing downstream of it can catch that" + ), + evidence_key="jump", + ), + DecisionClass( + name="hypothesis_generation", + decides="which candidate targets, conditions, or explanations are worth considering next", + required=Authority.MODEL, + because=( + "a hypothesis commits no material and no instrument time until something else schedules " + "it, and every class below this one stands between the proposal and an instrument. The " + "A-Lab is the worked example: its initial recipes came from natural-language models " + "trained on the literature, and what decided anything afterward was an active-learning " + "algorithm reading failed reactions, plus the furnace. Generation was safe precisely " + "because it was not authority" + ), + evidence_key="a_lab", + ), + DecisionClass( + name="protocol_drafting", + decides="the steps, order, reagents, and parameters of a procedure to be run", + required=Authority.MODEL_PROPOSES, + because=( + "a draft becomes an action the moment someone commits reagent and instrument time to " + "it, and the gap between a proposed route and an executable one is where the omissions " + "live. The Coley platform reached 15 drug or drug-like substances from AI-proposed " + "routes that expert chemists then had to turn into recipe files, supplying solvent " + "choice and precise stoichiometry among other implementation details, because the " + "chemical literature does not carry enough to get from inspiration to execution. The " + "ratifier need not be a person: a coded check that every reagent, volume, and " + "instrument named in the draft is declared in the registry is a ratifier, and it is " + "cheaper than a reviewer who is reading their fortieth draft that week" + ), + evidence_key="coley", + ), + DecisionClass( + name="capacity_forecast", + decides="how many plates, samples, or runs this lab can complete in a period", + required=Authority.MODEL_PROPOSES, + because=( + "a forecast moves nothing itself and is acted on by everyone, which is the worst " + "combination for a class with no ratifier. The deterministic check already exists in " + "this package: throughput refuses to total a protocol containing an UNKNOWN duration " + "and reports a floor from what was measured, so a drafted forecast can be mechanically " + "refused rather than argued with. AlabOS is the caution from the other side -- around " + "3,500 samples over about 1.5 years is a deployment usage statistic, and the scheduler " + "producing it is first-come-first-serve with no task-duration modeling at all, so " + "neither number forecasts anything about a different lab" + ), + evidence_key="alabos", + ), + DecisionClass( + name="schedule_proposal", + decides="the order queued work runs in, and which instrument takes which plate", + required=Authority.MODEL_PROPOSES, + because=( + "an ordering stays a proposal until a resource manager grants the device, and the grant " + "has to be deterministic because the constraint it enforces is physical rather than " + "preferential: two clients on one instrument is a USBError on the STAR and a silent " + "callback theft on the ODTC. AlabOS splits it exactly here -- a central resource " + "manager allocates devices and sample positions per task so that concurrent tasks " + "cannot conflict, and the framework explicitly does no experiment design of its own" + ), + evidence_key="alabos", + ), + DecisionClass( + name="exception_triage", + decides="what an unexpected observation means, and which recovery path follows from it", + required=Authority.MODEL_PROPOSES, + because=( + "triage is where fluency is most convincing and least checkable, because the input is " + "by definition off the distribution any rule anticipated. The mobile-robot chemistry " + "campaign is the case worth memorizing: one target underwent an unexpected " + "intramolecular cyclization whose product carried the same molecular weight as the " + "expected one and was not distinguishable by chromatogram or MS, so the workflow's " + "tiered heuristic let it through. Human inspection of the NMR caught it and " + "single-crystal diffraction settled it. A drafted triage is useful; a triage that " + "closes the exception on its own converts an anomaly into a result" + ), + evidence_key="dai", + ), + DecisionClass( + name="instrument_motion_limit", + decides="the travel, force, speed, and temperature bounds a machine may not exceed", + required=Authority.DETERMINISTIC, + because=( + "a limit is only a limit at the layer that can refuse. PyLabRobot draws its own " + "boundary in exactly the right place: its unit tests check that user input produces " + "the intended machine-level commands, and once the commands are sent they are " + "explicitly no longer the library's responsibility. A bound enforced by whatever " + "composes the command is a bound on the intent, so it is absent precisely when the " + "intent is wrong. Nothing that can be talked into a different answer belongs in this " + "path, which rules out a model in either role" + ), + evidence_key="pylabrobot", + ), + DecisionClass( + name="actuation_arming", + decides="whether the machine is permitted to move or energize at all, right now", + required=Authority.DETERMINISTIC, + because=( + "arming is the last point at which a wrong belief is still free. The interlock has to " + "answer from state a sensor reports -- door closed, plate seated, envelope clear -- " + "rather than from any account of the situation, because a coherent account of the " + "situation is exactly what a model produces when it is wrong. RoboCulture is the " + "stronger case and it is a strengthening, not a counterexample: the system detected " + "saturation and tracked growth across replicates, and the split was still manually " + "initiated as a safety precaution. HUMAN on a class whose floor is DETERMINISTIC is " + "permitted here and is never reported" + ), + evidence_key="roboculture", + ), + DecisionClass( + name="qc_result_disposition", + decides="whether a plate, run, or batch passes, and what happens to the material", + required=Authority.DETERMINISTIC, + because=( + "disposition is codeable, and where it is codeable it must be coded, because a " + "criterion in code can be versioned, argued with, and shown to a reviewer, while a " + "judgment call cannot be shown at all. The HTS assay validation guidance demonstrates " + "that these numbers exist and are writable -- a plate-uniformity study run over three " + "days, an acceptance criterion of signal window >= 2 or Z' >= 0.4, drift and edge " + "effects under about 20 percent -- and the same guidance says its replicate-experiment " + "study is a pre-production readiness tool and is not a substitute for post-production " + "monitoring. So DETERMINISTIC is the floor and not the end of the argument: qc.Gate is " + "the interlock, and every criterion in it still carries the Basis it rests on" + ), + evidence_key="ncats", + ), + DecisionClass( + name="batch_release", + decides="whether a result leaves this lab as something another party may rely on", + required=Authority.HUMAN, + because=( + "release is where this lab's uncertainty becomes someone else's assumption, and the " + "only mechanism anyone has made work for that is a named person who is answerable for " + "having released it. The drug CGMP data-integrity guidance is cited for the mechanism " + "and not for jurisdiction -- it is nonbinding guidance clarifying regulations a " + "research bench is not under -- and the mechanism is specific: production and control " + "records reviewed and approved by a quality unit, a system administrator role held " + "independently of the people responsible for record content, and data that cannot be " + "modified without a record of the modification. A model reporting that it checked " + "satisfies none of those, because none of them are about checking" + ), + evidence_key="fda_di", + ), + DecisionClass( + name="welfare_intervention", + decides="whether an animal is in distress, and what is done about it", + required=Authority.HUMAN, + because=( + "an automated readout can be genuinely useful here and still not be the decider, and " + "the multicentric home-cage study states both halves cleanly. Across 1,367 cages at " + "three institutions the model flagged animals 3 to 6 days before verifiable clinical " + "signs or death, at 66-80 percent accuracy on day -3 and 80-91 percent on day -6 " + "against a tolerated 15 percent false-positive budget. It also missed 9 to 20 percent " + "of verified clinical cases, measures locomotion only, cannot resolve individual " + "animals in a group-housed cage, delivers its alerts no earlier than the following " + "morning, and was analyzed retrospectively -- so nothing in it shows that acting on an " + "alert improved any outcome. A flag is evidence for a person. It is not a decision" + ), + evidence_key="homecage_multi", + ), + DecisionClass( + name="hazard_call", + decides="whether a material, reaction, or situation is dangerous, and what precaution follows", + required=Authority.HUMAN, + because=( + "this is the class the safety benchmark speaks to most directly, and being precise " + "about it matters more than the headline. Of 19 models evaluated, none exceeded 70 " + "percent accuracy on the open-ended hazards identification test, while the same " + "generation scored in the mid-80s on the multiple-choice portion. The gap is the " + "finding: structured recall does not carry into open-ended reasoning about a scenario, " + "and a scenario is the form a real hazard arrives in. What the benchmark measured is " + "constructed text questions at biosafety levels 1-3 aligned to US OSHA protocols, on a " + "late-2024 model snapshot -- no bench behavior, no incident rate. That is enough to " + "establish that a fluent hazard assessment is not evidence of one, which leaves " + "accountability as the only thing this class can rest on" + ), + evidence_key="labsafety", + ), + DecisionClass( + name="animal_procedure_authorization", + decides="whether a procedure on an animal may be performed at all", + required=Authority.HUMAN, + because=( + "authorization is held by a body, not by a document and not by a system. ARRIVE 2.0 is " + "a reporting guideline: its ethical-statement item asks authors to name the committee " + "that already approved the work, or to justify the absence of approval. Approval is " + "granted elsewhere and merely reported, so full compliance with the checklist says a " + "study was described transparently and says nothing about whether it was permitted or " + "sound. A pipeline that can produce the paperwork has produced a report, not " + "permission, and nothing in this package moves who holds it" + ), + evidence_key="arrive", + ), +) + + +DECISION_CLASSES_BY_NAME: Dict[str, DecisionClass] = {d.name: d for d in DECISION_CLASSES} + + +# -- a lab's actual plan, and what is wrong with it ------------------------------ + + +@dataclass(frozen=True) +class Assignment: + """One decision class, and the authority a lab has actually put on it. + + `who` is the named accountable party and matters only at HUMAN. It is checked by + `unaccountable()` rather than by `violations()`, because an unnamed human approver is a + different defect from a weakened authority: the authority is right and the accountability + is missing, so the plan reads compliant and no one is answerable. + """ + + decision_class: str + authority: Authority + who: str = "" + note: str = "" + + +@dataclass(frozen=True) +class Plan: + """What a lab intends to let its models decide. The object this module checks. + + Nothing here can declare itself compliant. There is no field an author can set that + suppresses a violation, for the same reason a Gate has no `ready` field: a report that can + be silenced is a report about the author rather than the lab. + """ + + name: str + assignments: Tuple[Assignment, ...] = () + + def assigned(self, decision_class: str) -> Optional[Assignment]: + for a in self.assignments: + if a.decision_class == decision_class: + return a + return None + + @property + def escalation_only(self) -> bool: + """True when no assignment in this plan sits below the authority its class requires. + + Necessary and nowhere near sufficient, and the difference is the point. An empty plan is + trivially escalation-only because it weakens nothing, and it covers nothing at all. Use + `complies` for the question a lab actually wants answered. + """ + return not any(v.kind is ViolationKind.WEAKENED for v in violations(self)) + + @property + def complies(self) -> bool: + """No class is weakened, and no class is left unplaced.""" + return not violations(self) + + +class ViolationKind(str, Enum): + """The three ways a plan fails, which need three different fixes.""" + + WEAKENED = "weakened" # an authority below what the class requires + UNASSIGNED = "unassigned" # the class exists and the plan is silent about it + UNDECLARED = "undeclared" # the plan places a class this registry does not declare + + +@dataclass(frozen=True) +class Violation: + """One place a plan does not hold, with enough detail to act on it.""" + + decision_class: str + kind: ViolationKind + required: Optional[Authority] + assigned: Optional[Authority] + reason: str + evidence_key: Optional[str] = None + + @property + def model_overreach(self) -> bool: + """A model was given a decision that requires an interlock or a person. + + The specific defect the module was written for, separated from the others so a report + can lead with it. Everything else here is a gap in the plan; this is a claim about the + lab that is false. + """ + return ( + self.kind is ViolationKind.WEAKENED + and self.assigned is not None + and self.assigned.model_in_path + and self.required is not None + and not self.required.model_in_path + ) + + +def violations( + plan: Plan, + classes: Optional[Sequence[DecisionClass]] = None, +) -> Tuple[Violation, ...]: + """Every place `plan` assigns an authority weaker than the class requires, plus every + class it does not assign at all. + + Iterates the CLASSES, not the assignments. Written the other way round -- walking the + plan and checking each entry -- it returns clean for a plan that mentions two classes and + omits eleven, which is the shape every real authority document has. Absence of an + assignment is not compliance, and the omitted classes are systematically the ones nobody + thought about, so they are exactly the ones worth naming. + """ + registry = tuple(classes) if classes is not None else DECISION_CLASSES + out: List[Violation] = [] + + for dc in registry: + assignment = plan.assigned(dc.name) + if assignment is None: + out.append( + Violation( + decision_class=dc.name, + kind=ViolationKind.UNASSIGNED, + required=dc.required, + assigned=None, + reason=( + f"the plan says nothing about who decides '{dc.decides}'; this class requires " + f"{dc.required.value} and an unassigned class is not a permitted one, it is an " + f"unexamined one" + ), + evidence_key=dc.evidence_key, + ) + ) + continue + if not dc.permits(assignment.authority): + out.append( + Violation( + decision_class=dc.name, + kind=ViolationKind.WEAKENED, + required=dc.required, + assigned=assignment.authority, + reason=( + f"'{dc.name}' is assigned {assignment.authority.value} and requires " + f"{dc.required.value}: {dc.because}" + ), + evidence_key=dc.evidence_key, + ) + ) + + declared = {dc.name for dc in registry} + for a in plan.assignments: + if a.decision_class not in declared: + out.append( + Violation( + decision_class=a.decision_class, + kind=ViolationKind.UNDECLARED, + required=None, + assigned=a.authority, + reason=( + f"the plan assigns '{a.decision_class}', which no decision class declares; " + f"either the registry is missing a real decision or the plan is placing " + f"authority on something nobody defined" + ), + ) + ) + + return tuple(out) + + +def strengthened( + plan: Plan, + classes: Optional[Sequence[DecisionClass]] = None, +) -> Tuple[Tuple[str, Authority, Authority], ...]: + """(class, required, assigned) wherever a lab went above the floor. + + Reported and never flagged, which is the asymmetry the whole module rests on. It is worth + seeing, though: a lab that has strengthened eleven of thirteen classes has bought a lot of + approvals, and approval fatigue is a real failure mode that nothing here measures. + """ + registry = tuple(classes) if classes is not None else DECISION_CLASSES + out: List[Tuple[str, Authority, Authority]] = [] + for dc in registry: + a = plan.assigned(dc.name) + if a is not None and a.authority.rank > dc.required.rank: + out.append((dc.name, dc.required, a.authority)) + return tuple(out) + + +def unaccountable( + plan: Plan, + classes: Optional[Sequence[DecisionClass]] = None, +) -> Tuple[str, ...]: + """Classes placed at HUMAN with nobody named. + + A human in the loop who is not named is not accountable; it is a queue. Kept out of + `violations()` because the authority level is correct and only the accountability is + missing, so folding them together would let a fix for one look like a fix for the other. + """ + registry = tuple(classes) if classes is not None else DECISION_CLASSES + declared = {dc.name for dc in registry} + return tuple( + a.decision_class + for a in plan.assignments + if a.decision_class in declared and a.authority is Authority.HUMAN and not a.who.strip() + ) + + +# -- the qualification ladder ----------------------------------------------------- +# A separate structure from authority, and deliberately not merged with it. Authority asks +# who may decide; the ladder asks what has been PROVEN about a capability. A lab can get +# both wrong independently, and conflating them produces the most common overclaim in this +# field: a system that passed a simulation, is described as validated, and is then handed a +# decision that requires an interlock. +# +# A rung is not a Verdict, and the two answer different questions. Verdict says what the +# machine can do today -- BROKEN, MANUAL, WRITTEN, BLOCKED, SUPERVISED, AUTOMATED. A rung +# says what anyone has established about doing it. WRITTEN is the verdict of a step standing +# on SIMULATION and nothing above it: dry-validated, never on the instrument. AUTOMATED is +# not a rung at all -- a step can run headless every night and stand on no rung, which is +# precisely the gap intelligence.Benchmark exists to name and the reason trusted_for() +# answers no for an operation nobody measured. + + +class Rung(str, Enum): + """The qualification ladder, bottom to top. A rung cannot be skipped.""" + + SIMULATION = "simulation" + DEVICE_ACCEPTANCE = "device_acceptance" + INTEGRATED_DRY_RUN = "integrated_dry_run" + VOLUMETRIC_QUALIFICATION = "volumetric_qualification" + ASSAY_TRANSFER = "assay_transfer" + SUSTAINED_PRODUCTION = "sustained_production" + + @property + def height(self) -> int: + return _RUNG_ORDER.index(self) + + @property + def physical(self) -> bool: + """True for every rung that requires the machine to have moved real material. + + The line the ladder exists to hold. SIMULATION is below it, and everything a simulation + establishes is a statement about generated commands rather than about matter. + """ + return self is not Rung.SIMULATION + + +_RUNG_ORDER: Tuple[Rung, ...] = ( + Rung.SIMULATION, + Rung.DEVICE_ACCEPTANCE, + Rung.INTEGRATED_DRY_RUN, + Rung.VOLUMETRIC_QUALIFICATION, + Rung.ASSAY_TRANSFER, + Rung.SUSTAINED_PRODUCTION, +) + + +@dataclass(frozen=True) +class QualificationRung: + """One rung, what passing it establishes, and what it explicitly does not. + + `does_not_qualify` is required for the same reason `because` is required on a decision + class. A rung stated without its limit is how a real result becomes an overclaim: nobody + fabricates the tip-insertion rate, they just let it stand in for the assay. + """ + + rung: Rung + qualifies: str + does_not_qualify: str + evidence_key: Optional[str] = None + + def __post_init__(self) -> None: + if not self.does_not_qualify.strip(): + raise ValueError( + f"rung '{self.rung.value}' states no limit; a qualification with no stated limit " + f"will be read as qualifying everything above it" + ) + + +LADDER: Tuple[QualificationRung, ...] = ( + QualificationRung( + rung=Rung.SIMULATION, + qualifies=( + "that the commands a protocol generates are the ones intended, and that the sequence " + "is internally coherent against a deck model" + ), + does_not_qualify=( + "anything the machine does with them. PyLabRobot states the boundary itself: its unit " + "tests establish that user input produces consistent machine-level commands, and after " + "the commands are transmitted they are explicitly no longer the library's " + "responsibility. Its simulator is designed to work with any robot for which a deck " + "model exists, and the authors say plainly that some development still requires " + "physical access to a robot. No accuracy or precision data is reported anywhere in " + "that work, so nothing at this rung is a statement about volume in a well" + ), + evidence_key="pylabrobot", + ), + QualificationRung( + rung=Rung.DEVICE_ACCEPTANCE, + qualifies=( + "that one device performs the mechanical action it claims, repeatably, measured on " + "that device in that configuration" + ), + does_not_qualify=( + "the assay, another device, or the same device in a different cell. RoboCulture " + "reports 99.0 percent pipette-tip insertion across 576 attempts at six random plate " + "placements -- a real number about a manipulator that says nothing about what ends up " + "in the well. Its perception depends on fiducial tags, assumes they remain unoccluded, " + "requires a calibration the authors call fragile to maintain, and is stated not to " + "generalize beyond well plates" + ), + evidence_key="roboculture", + ), + QualificationRung( + rung=Rung.INTEGRATED_DRY_RUN, + qualifies=( + "that stations, transfers, scheduling, and error paths survive an end-to-end pass with " + "no material at risk" + ), + does_not_qualify=( + "wet behavior, or any failure class the dry run did not contain. AlabOS is explicit " + "that failures from worn-out hardware and newly emerged software communication " + "exceptions were not detected in its soak test and still required operators, and its " + "user-intervention module -- dashboard, Slack, email prompts for robot-arm error " + "recovery and consumable replacement -- is load-bearing rather than vestigial. " + "Autonomy at this rung means orchestration, not unattended operation" + ), + evidence_key="alabos", + ), + QualificationRung( + rung=Rung.VOLUMETRIC_QUALIFICATION, + qualifies=( + "that delivered volume has been determined and reported for this installed system by a " + "stated procedure, with the statistics indexed and tracked" + ), + does_not_qualify=( + "acceptance. ISO 23783-1 covers vocabulary and general requirements and 23783-3 covers " + "determination, specification and reporting of volumetric performance; the measurement " + "procedures themselves are in Part 2, which is a separate document. None of them set a " + "pass/fail tolerance for any particular assay, none certify an instrument, and the " + "series is voluntary and scoped to complete installed systems that handle liquid " + "without human intervention. It also addresses volume only -- not carryover, not " + "sterility, not whether the assay means anything. The lab still has to say what volume " + "error its assay tolerates, and that is a scientific claim carrying a Basis" + ), + evidence_key="iso23783", + ), + QualificationRung( + rung=Rung.ASSAY_TRANSFER, + qualifies=( + "that this assay performs on this system, against criteria fixed before the data were " + "collected" + ), + does_not_qualify=( + "production. The HTS assay validation guidance is direct about its own scope: the " + "replicate-experiment study is a diagnostic and decision tool for establishing that an " + "assay is ready to go into production, and is explicitly not intended as a substitute " + "for post-production monitoring. Its plate-uniformity design is three days of plates " + "and its recommended acceptance criterion is signal window >= 2 or Z' >= 0.4 -- that " + "manual's recommendation for HTS screening in drug discovery, not a universal quality " + "bar and not a regulatory validation. Passing it says the assay is fit to screen; it " + "says nothing about whether any hit it finds is real" + ), + evidence_key="ncats", + ), + QualificationRung( + rung=Rung.SUSTAINED_PRODUCTION, + qualifies=( + "that the system held together across a real campaign, under the conditions of that " + "campaign" + ), + does_not_qualify=( + "a rate, or the next campaign, or the quality of what came out. The A-Lab realized 36 " + "compounds from 57 targets over 17 days of continuous operation: one uncontrolled run " + "on one instrument, no baseline against human chemists and no replicate campaign, so 63 " + "percent describes that run and is not a capability benchmark. The same work is the " + "reason this rung stays separate from the one below it -- four reported successes were " + "later reclassified as inconclusive from diffraction alone, the title's claim of " + "novelty was withdrawn in correction, and the senior author concedes a human can " + "perform a higher-quality refinement than the automated pipeline did" + ), + evidence_key="a_lab", + ), +) + + +LADDER_BY_RUNG: Dict[Rung, QualificationRung] = {r.rung: r for r in LADDER} + + +def implies(passed: Rung, claimed: Rung) -> bool: + """Does passing `passed` establish `claimed`? + + True only when `claimed` sits at or below `passed`. Passing rung N never implies rung + N+1, and the reason is not caution: each rung is qualified by a different measurement on a + different thing. A simulation measures generated commands, device acceptance measures a + mechanism, volumetric qualification measures delivered liquid. There is no inference from + one to the next, only a further experiment. + """ + return claimed.height <= passed.height + + +def qualified_for(passed: Sequence[Rung]) -> Tuple[Rung, ...]: + """The rungs actually supported by a set of passed rungs: the contiguous run from the + bottom, and nothing above the first gap. + + A lab that has passed simulation and assay transfer but never qualified volume is + qualified for simulation. The assay-transfer result is not discarded -- it just does not + stand on anything, and a result that does not stand on anything is not a qualification, + because the thing it would have to be true of has not been characterized. + """ + have = set(passed) + out: List[Rung] = [] + for rung in _RUNG_ORDER: + if rung not in have: + break + out.append(rung) + return tuple(out) + + +def missing_below(claimed: Rung, passed: Sequence[Rung]) -> Tuple[Rung, ...]: + """Every rung below `claimed` that has not been passed. Empty means the claim stands.""" + have = set(passed) + return tuple(r for r in _RUNG_ORDER if r.height < claimed.height and r not in have) + + +def unsupported_claims( + claimed: Sequence[Rung], + passed: Sequence[Rung], +) -> Tuple[Tuple[Rung, Tuple[Rung, ...]], ...]: + """(claim, the rungs missing under it) for every claim a lab cannot support. + + A claimed rung that was never itself passed counts as missing under itself, so claiming a + rung on the strength of the one below it is caught by the same computation that catches + skipping. + """ + out: List[Tuple[Rung, Tuple[Rung, ...]]] = [] + have = set(passed) + for c in claimed: + gaps = missing_below(c, passed) + if c not in have: + gaps = gaps + (c,) + if gaps: + out.append((c, tuple(sorted(gaps, key=lambda r: r.height)))) + return tuple(out) + + +__all__ = [ + "Assignment", + "Authority", + "DECISION_CLASSES", + "DECISION_CLASSES_BY_NAME", + "DecisionClass", + "LADDER", + "LADDER_BY_RUNG", + "Plan", + "QualificationRung", + "Rung", + "Violation", + "ViolationKind", + "implies", + "missing_below", + "qualified_for", + "strengthened", + "unaccountable", + "unsupported_claims", + "violations", +] diff --git a/autonomous_lab/cadence.py b/autonomous_lab/cadence.py new file mode 100644 index 0000000..2afe32d --- /dev/null +++ b/autonomous_lab/cadence.py @@ -0,0 +1,1041 @@ +"""Turning "N plates a day" into the interval a plate must actually enter, and what that forces. + +A throughput target arrives as a round number. Five hundred plates a day. It is stated in +plates and days because those are the units a budget is written in, and it is almost never +converted into the unit a workcell is actually built against, which is SECONDS BETWEEN +ADMISSIONS. The conversion is one division. It needs no measured durations, nothing about +it can be argued with, and it is consistently sobering: 500 a day is a plate entering every +172.8 seconds, sustained, including nights, weekends, and the morning after a jam. + +That interval is the specification. Deck layout, buffer positions, arm reach, how many +people are on shift, how long a tip refill may take -- all of it is downstream of it, and a +lab that never computed it is designing against a number nobody wrote down. + +Two figures are reported rather than one. The same 500 plates inside a staffed 16-hour day +is a plate every 115.2 seconds, which is 1.5 times tighter for no change in the target. That +ratio IS the argument for walk-away capability, stated in the only terms that make it an +argument rather than a preference: every hour the lab is not admitting plates, the hours it +is must go proportionally faster, and the penalty is multiplicative rather than additive. + +THE TARGET HAS TO SAY WHAT IT COUNTS, and this module refuses one that does not. "Five +hundred plates a day" is four different specifications: + + STARTED plates admitted to the workcell. + COMPLETED plates that reached the last step. + QC_PASSED plates that reached the last step and passed their gates. + RELEASED plates whose result was released for use. + +Only the first is a rate a scheduler controls. The other three are outputs, and converting +an output target back into an admission interval needs the failure-and-rerun rate between +them, which is a measured quantity and not a rounding. At a 20 percent loss, 500 RELEASED +is an admission rate of 625 and an interval under 139 seconds rather than 172.8. Answering +the wrong one of the four is not a refinement; it is the difference between a workcell that +meets its target and one that misses by a quarter and cannot say which step ate it. So +`cadence_for` and `demonstrated_support` both refuse a target with no stated endpoint +instead of quietly picking the flattering reading. + +WHAT THE INTERVAL FORCES is the second half. `implied_constraints` computes four things, +and the split between them is the load-bearing one: two are purchases and two are charges +against the same clock. Parallel LINES and BUFFER capacity are relieved by hardware -- a +second line, more nests. RECOVERY and UPKEEP are not. A stoppage at a sustained cadence +costs admissions at exactly the launch rate and there is no idle time in the day to make +them back, so recovering an hour means running above target afterward, which is a higher +cadence than the one anybody costed. Tips, reagents, waste, calibration, and record review +are charged against the identical window, and no published source sizes that charge. + +`headroom` is where "just buy a faster robot" dies. If a human is needed on some fraction +of plates for some number of minutes, that consumes a fixed share of every admission +interval, and instrument speed appears nowhere in the arithmetic. Above 100 percent the +queue grows without bound and the target is unreachable at any hardware budget. + +The last section is the honest check, and it mirrors what `throughput` does with a +makespan. `throughput` refuses to return plates per day because 18 of 18 steps have never +been timed, and that refusal stands. This module can compute an interval from arithmetic +alone, so it has nothing to refuse there -- but it must not let arithmetic be mistaken for +evidence that the rate is achievable. `demonstrated_support` answers the question that +actually matters after the division: has anybody published a system that sustains this? + +Across every work cited here the answer is no, and none of them is close. The longest +continuously-operating integrated system in the list ran 17 days, in one domain, on +inorganic powders. The only citation reporting a per-day number reports a PEAK day, from a +deployment whose own authors call its scheduler non-optimal. The one work that does operate +continuously at scale across three institutions for over a thousand days is a CAGE SENSOR: +it observes, it does not actuate, its alerts are held until morning, and the analysis is +retrospective. None of that is a production rate. Every citation below is recorded with +what it does not establish, because rounding a demonstration up into a benchmark is the +exact failure this package exists to refuse. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from enum import Enum +from typing import Dict, List, Optional, Tuple + +SECONDS_PER_HOUR = 3600.0 +SECONDS_PER_DAY = 24 * SECONDS_PER_HOUR + + +class Endpoint(str, Enum): + """What a plates-per-day target counts. Four different specifications, one phrase. + + The line this enum draws is between a rate the scheduler SETS and a rate the workcell + RETURNS. Admissions are set: the scheduler decides when a plate enters and nothing + downstream can change that decision after the fact. The other three are returns, and each + one is the previous one multiplied by a survival fraction -- steps that abort, gates that + fail, results withheld. A lab that states a target without saying which of these it means + has not specified anything, because the four differ by exactly the rate at which its work + fails, and that rate is a measurement rather than an assumption. + """ + + STARTED = "started" # admitted to the workcell + COMPLETED = "completed" # reached the last step + QC_PASSED = "qc_passed" # reached the last step and passed its gates + RELEASED = "released" # result released for use + + @property + def is_admission_rate(self) -> bool: + """True only for STARTED: the one endpoint an interval computes directly. + + For every other endpoint the interval is strictly shorter than the naive division, by + a factor nobody in this repo has measured. `intelligence.knowledge_summary` reports + almost every benchmark as UNMEASURED, so the survival fraction is not merely unknown + here, it is unknowable from what this lab has recorded. + """ + return self is Endpoint.STARTED + + +@dataclass(frozen=True) +class Target: + """A throughput target as it was actually stated, including what it left out. + + `endpoint` defaults to None on purpose rather than to STARTED. A default would resolve + the ambiguity silently and in the direction that flatters the number -- STARTED is always + the loosest of the four -- and the whole point of this dataclass is that the ambiguity + survives long enough to be refused. + """ + + plates_per_day: float + endpoint: Optional[Endpoint] = None + note: str = "" + + def __post_init__(self) -> None: + if self.plates_per_day <= 0: + raise ValueError( + f"a target of {self.plates_per_day} plates/day is not a target; an interval over it " + "is a division by zero or a negative clock" + ) + + @property + def ambiguous(self) -> bool: + return self.endpoint is None + + def why_ambiguous(self) -> str: + """The refusal, with the four readings named so the caller can pick one.""" + return ( + f"a target of {self.plates_per_day:g} plates/day does not say what it counts. It could " + "mean plates STARTED, plates COMPLETED, plates QC_PASSED, or plates RELEASED, and those " + "four differ by the failure and rerun rate between them. Only STARTED is an admission " + "rate an interval computes directly; the rest need a measured survival fraction this " + "lab does not have. State the endpoint rather than accepting the loosest reading" + ) + + +def launch_interval(plates_per_day: float, operating_hours: float = 24.0) -> float: + """Seconds between plate admissions to sustain a rate inside a window. + + One division, and the only function here that needs no evidence of any kind. It is + deliberately a function on two numbers rather than a method on `Target`: the arithmetic is + true whatever the target means, and the endpoint refusal belongs one layer up, where the + number becomes a claim about a lab rather than a quotient. + + The result is an ADMISSION interval. If the caller's target counts anything downstream of + admission, this is a ceiling on the true interval and the real one is shorter -- see + `required_admissions`. + """ + if plates_per_day <= 0: + raise ValueError(f"{plates_per_day} plates/day has no launch interval") + if operating_hours <= 0: + raise ValueError(f"{operating_hours} operating hours is not a window") + if operating_hours > 24: + raise ValueError( + f"{operating_hours} operating hours does not fit in a day; a window longer than the " + "clock it is measured against would make the interval look achievable by arithmetic" + ) + return (operating_hours * SECONDS_PER_HOUR) / plates_per_day + + +def intervals(plates_per_day: float, operating_hours: float) -> Tuple[float, float]: + """(round-the-clock seconds, staffed-hours seconds) for the same target. + + Both, always, because the gap between them is the argument. Quoting only the 24-hour + figure describes a lab that runs unattended, and quoting only the staffed figure hides + what unattended running would buy. + """ + return ( + launch_interval(plates_per_day, 24.0), + launch_interval(plates_per_day, operating_hours), + ) + + +def required_admissions(target: Target, yield_fraction: Optional[float] = None) -> Optional[float]: + """Plates that must be ADMITTED per day to hit a target stated on some endpoint. + + Returns None for a downstream endpoint with no declared yield, which is the honest answer + rather than the convenient one. The convenient answer is to assume a yield of 1.0, and it + is wrong in a specific direction: it under-admits, so the workcell misses the target by + exactly the failure rate and the shortfall gets blamed on the instruments. + """ + if target.ambiguous: + raise ValueError(target.why_ambiguous()) + if target.endpoint is not None and target.endpoint.is_admission_rate: + return target.plates_per_day + if yield_fraction is None: + return None + if not 0 < yield_fraction <= 1: + raise ValueError(f"a yield of {yield_fraction} is not a fraction of plates that survive") + return target.plates_per_day / yield_fraction + + +class Forcing(str, Enum): + """What a sustained admission interval forces on a workcell. + + The line here is whether hardware relieves it, and it splits the four evenly. LINES and + BUFFER are capital: a second line, a deeper hotel, a faster slowest step. RECOVERY and + UPKEEP are charges against the same window, and no purchase order touches them -- a + stoppage costs admissions at the launch rate whatever the instrument is capable of, and + tips still have to be loaded, waste emptied, and audit trails reviewed inside the identical + 24 hours. Drawing the line in the enum rather than in prose is deliberate: a workcell that + misses its target buys hardware by reflex, and two of these four say the money will not help. + """ + + LINES = "lines" # parallel lines, because the interval is shorter than a single step + BUFFER = "buffer" # plates resident at once, which the deck must physically hold + RECOVERY = "recovery" # admissions lost per unit of downtime, unrecoverable at a sustained rate + UPKEEP = "upkeep" # maintenance, replenishment and record review, inside the same window + + @property + def relieved_by_hardware(self) -> bool: + return self in (Forcing.LINES, Forcing.BUFFER) + + +@dataclass(frozen=True) +class Implication: + """One thing an admission interval forces, with the sum shown so it can be checked. + + `computable` is the field to read first, and it is False whenever the caller declared no + duration for the quantity involved. That is the same refusal `throughput` makes: the + reference protocol has no measured step durations at all, so a line count or a buffer + depth over them would be a capital plan resting on a guess. RECOVERY and UPKEEP are always + computable because they are arithmetic on the interval alone. + """ + + forcing: Forcing + statement: str + arithmetic: str + computable: bool + value: Optional[float] = None + unachievable: bool = False # one line cannot meet this target with the declared step + + @property + def relieved_by_hardware(self) -> bool: + return self.forcing.relieved_by_hardware + + +@dataclass(frozen=True) +class Cadence: + """A target converted into the interval a plate must enter, and what that costs. + + Carries both intervals rather than one, plus whatever durations the caller was willing to + declare. The implied constraints are a METHOD rather than a stored field, for the same + reason the ledger computes verdicts instead of storing them: a derived value kept beside + the values it derives from eventually disagrees with them, and nothing reveals which one + is wrong. Everything here recomputes from `interval_seconds` on demand. + """ + + target: Target + operating_hours: float + interval_seconds: float # inside the staffed window + round_the_clock_seconds: float # the same target spread over all 24 hours + slowest_step_seconds: Optional[float] = None + end_to_end_seconds: Optional[float] = None + + @property + def window_seconds(self) -> float: + return self.operating_hours * SECONDS_PER_HOUR + + @property + def minutes_between_plates(self) -> float: + return self.interval_seconds / 60.0 + + @property + def walk_away_multiplier(self) -> float: + """How much tighter the staffed interval is than the round-the-clock one. + + Exactly 24 / operating_hours, which is the point: the penalty for not running unattended + does not depend on the target, the protocol, or the instruments. It is a property of the + clock, and it is the whole quantitative case for closing the steps that need a human. + """ + return self.round_the_clock_seconds / self.interval_seconds + + @property + def lines_required(self) -> Optional[int]: + """Parallel lines needed, or None when no slowest step was declared.""" + if self.slowest_step_seconds is None: + return None + return max(1, math.ceil(self.slowest_step_seconds / self.interval_seconds)) + + @property + def single_line_feasible(self) -> Optional[bool]: + """False when the interval is shorter than the slowest declared step. + + Not a scheduling result. A single line admits at most one plate per slowest-step + duration, so a target below that is unachievable on one line no matter how fast every + other instrument becomes, and no buffer, scheduler, or plate mover lowers the floor. + """ + lines = self.lines_required + return None if lines is None else lines <= 1 + + @property + def plates_in_flight(self) -> Optional[int]: + """Plates resident in the workcell at once, or None with no end-to-end duration.""" + if self.end_to_end_seconds is None: + return None + return max(1, math.ceil(self.end_to_end_seconds / self.interval_seconds)) + + def admissions_lost(self, stoppage_seconds: float) -> float: + """Admissions given up by stopping for this long. + + Fractional on purpose. Rounding down would suggest a short stop is free, and at a + sustained target it is not: the day has no idle time to absorb it, so a fraction of a + plate lost today is a whole plate lost by the third stop. + """ + if stoppage_seconds < 0: + raise ValueError("a stoppage cannot be negative") + return stoppage_seconds / self.interval_seconds + + def interval_net_of(self, upkeep_seconds: float) -> float: + """The interval once upkeep has taken its share of the same window. + + Refuses upkeep that consumes the window rather than returning a negative or infinite + interval, because a negative interval formats as a number and would be quoted as one. + """ + if upkeep_seconds < 0: + raise ValueError("upkeep cannot be negative") + remaining = self.window_seconds - upkeep_seconds + if remaining <= 0: + raise ValueError( + f"{upkeep_seconds:g} s of upkeep consumes the whole {self.window_seconds:g} s window; " + "there is no interval left to admit a plate at, and the target is unreachable before " + "any instrument is considered" + ) + return remaining / self.target.plates_per_day + + def implied_constraints(self) -> List[Implication]: + """The four things this interval forces, computed or explicitly refused. + + Order is fixed and every forcing appears exactly once, including the ones that had to be + refused. A report that omits an unanswerable constraint reads as though it does not + apply, which is the opposite of what a missing duration means. + """ + return [ + self._lines(), + self._buffer(), + self._recovery(), + self._upkeep(), + ] + + # -- the four forcings ------------------------------------------------------- + + def _lines(self) -> Implication: + if self.slowest_step_seconds is None: + return Implication( + forcing=Forcing.LINES, + statement=( + "no slowest-step duration was declared, so the line count is not computable. That " + "is not a gap in this model: `throughput` reports every step of the reference " + "protocol as UNKNOWN, and a line count over guessed durations is a capital plan " + "resting on a guess" + ), + arithmetic="ceil(slowest step / interval) -- the numerator does not exist", + computable=False, + ) + slowest = self.slowest_step_seconds + lines = self.lines_required or 1 + arithmetic = f"ceil({slowest:g} s / {self.interval_seconds:.1f} s) = {lines} line(s)" + if lines > 1: + statement = ( + f"one line holds a plate at its slowest step for {slowest:g} s, so it admits at most " + f"one plate per {slowest:g} s. The target needs one every {self.interval_seconds:.1f} s, " + f"which is unachievable on a single line: {lines} lines must run in parallel. Speeding " + "up every OTHER instrument changes nothing, because a single line's admission interval " + "is floored by its slowest step and no scheduler, buffer, or plate mover lowers that floor" + ) + else: + statement = ( + f"the slowest declared step holds a line for {slowest:g} s, which fits inside the " + f"{self.interval_seconds:.1f} s admission interval, so one line is arithmetically " + "sufficient. That is a claim about one step and nothing else -- it assumes the caller " + "named the true slowest step and that no two steps contend for the same instrument, " + "which `orchestrate` computes separately and which the one-process-per-instrument rule " + "makes the common case" + ) + return Implication( + forcing=Forcing.LINES, + statement=statement, + arithmetic=arithmetic, + computable=True, + value=float(lines), + unachievable=lines > 1, + ) + + def _buffer(self) -> Implication: + if self.end_to_end_seconds is None: + return Implication( + forcing=Forcing.BUFFER, + statement=( + "no end-to-end duration was declared, so the number of plates resident at once is " + "not computable. `throughput` refuses a makespan for the same reason, and a buffer " + "depth is a makespan divided by an interval -- it inherits every unmeasured second" + ), + arithmetic="ceil(end to end / interval) -- the numerator does not exist", + computable=False, + ) + e2e = self.end_to_end_seconds + resident = self.plates_in_flight or 1 + return Implication( + forcing=Forcing.BUFFER, + statement=( + f"a plate is inside the workcell for {e2e:g} s end to end. At an admission every " + f"{self.interval_seconds:.1f} s that is {resident} plates resident simultaneously, so " + f"the decks, nests and hotel must physically hold {resident} plates and the record must " + f"tell all {resident} apart -- `provenance` counts an unobserved custody hop for every " + "one it cannot. ISO 23783-1:2022 and ISO 23783-3:2022 specify automated liquid handler " + "vocabulary and volumetric performance reporting; neither states a rate or a capacity, " + "so no published standard sizes this for you" + ), + arithmetic=f"ceil({e2e:g} s / {self.interval_seconds:.1f} s) = {resident} plates resident", + computable=True, + value=float(resident), + ) + + def _recovery(self) -> Implication: + per_minute = 60.0 / self.interval_seconds + return Implication( + forcing=Forcing.RECOVERY, + statement=( + f"a stoppage costs {per_minute:.2f} admissions for every minute stopped, and a " + "SUSTAINED target has no idle time to make them back. Recovering an hour means running " + "above the target afterward, which is a higher cadence than the one anybody costed, and " + "it compounds with whatever caused the stoppage. Published autonomous runs report " + "recovery as human work: AlabOS carries an explicit user-request module for robot-arm " + "error recovery and consumable replacement, and its authors state that worn hardware and " + "communication exceptions not caught in soak testing still require operators" + ), + arithmetic=( + f"60 s / {self.interval_seconds:.1f} s = {per_minute:.2f} admissions lost per minute" + ), + computable=True, + value=per_minute, + ) + + def _upkeep(self) -> Implication: + per_hour = SECONDS_PER_HOUR / self.interval_seconds + return Implication( + forcing=Forcing.UPKEEP, + statement=( + "tips, reagents, waste, calibration and record review are charged against the same " + f"window as the plates. Every {self.interval_seconds:.1f} s of upkeep inside the " + f"{self.operating_hours:g} h window costs exactly one admission, so an hour of daily " + f"upkeep costs {per_hour:.1f} plates. The FDA data integrity guidance expects audit " + "trails to be reviewed and deliberately prescribes no frequency, deferring to the firm's " + "own risk assessment -- so the charge is real, it recurs every day, and no published " + "source sizes it. It has to come out of this window or out of the target" + ), + arithmetic=( + f"3600 s / {self.interval_seconds:.1f} s = {per_hour:.1f} plates per hour of upkeep" + ), + computable=True, + value=per_hour, + ) + + +def cadence_for( + target, + operating_hours: float = 24.0, + slowest_step_seconds: Optional[float] = None, + end_to_end_seconds: Optional[float] = None, +) -> Cadence: + """Convert a stated target into a sustained admission interval. + + Accepts a `Target` or a bare number, and refuses the bare number for the same reason it + refuses a `Target` with no endpoint: a plain float carries no statement of what it counts, + so accepting one would be an undocumented route around the refusal rather than a + convenience. + """ + t = target if isinstance(target, Target) else Target(plates_per_day=float(target)) + if t.ambiguous: + raise ValueError(t.why_ambiguous()) + for name, value in ( + ("slowest_step_seconds", slowest_step_seconds), + ("end_to_end_seconds", end_to_end_seconds), + ): + if value is not None and value <= 0: + raise ValueError(f"{name} of {value} is not a duration") + staffed = launch_interval(t.plates_per_day, operating_hours) + return Cadence( + target=t, + operating_hours=operating_hours, + interval_seconds=staffed, + round_the_clock_seconds=launch_interval(t.plates_per_day, 24.0), + slowest_step_seconds=slowest_step_seconds, + end_to_end_seconds=end_to_end_seconds, + ) + + +# -- the human share of the clock ---------------------------------------------- + + +@dataclass(frozen=True) +class Headroom: + """How much of the admission interval a human consumes, and where the target dies. + + Note what is absent from every field here: instrument speed. The human seconds per plate + are set by how often somebody is needed and how long they take; the interval is set by the + target. A faster robot moves neither, so a workcell that fails this check fails it at any + hardware budget. That is the whole reason this computation exists separately from + `orchestrate`, which already reports WHERE a human is needed but not what it costs per plate. + """ + + interval_seconds: float + intervention_rate: float # interventions per plate admitted + minutes_per_intervention: float + human_seconds_per_plate: float + consumed_fraction: float + + @property + def reachable(self) -> bool: + """False at or above 100 percent of the clock. + + Exactly 1.0 is not reachable, deliberately. A queue whose service time equals its + interarrival time has zero slack, and any variance at all -- and instrument timings are + bimodal, which `throughput` records for the one step this repo has actually measured -- + makes the backlog grow without bound rather than hover. + """ + return self.consumed_fraction < 1.0 + + @property + def slack_seconds(self) -> float: + """Seconds of the interval left after the human. Negative when the target is unreachable.""" + return self.interval_seconds - self.human_seconds_per_plate + + @property + def operators_required(self) -> int: + """Operators needed in parallel to hold this cadence. + + Assumes interventions on different plates can proceed simultaneously, which is false + wherever they contend for the same instrument -- `orchestrate` already serializes those + under the one-process-per-instrument rule. So this is a FLOOR on the headcount, not an + estimate of it. + """ + if self.human_seconds_per_plate <= 0: + return 0 + return max(1, math.ceil(self.consumed_fraction)) + + @property + def ceiling_interval_seconds(self) -> Optional[float]: + """The shortest interval one operator can sustain. None when no human is needed.""" + if self.human_seconds_per_plate <= 0: + return None + return self.human_seconds_per_plate + + def ceiling_plates_per_day(self, operating_hours: float = 24.0) -> Optional[float]: + """The most plates one operator can sustain inside a window, ignoring everything else. + + A strict upper bound and nothing more: it grants the operator every second of the window + with no breaks, no handover, and no second task, which no shift has ever looked like. + """ + if self.human_seconds_per_plate <= 0: + return None + if operating_hours <= 0 or operating_hours > 24: + raise ValueError(f"{operating_hours} operating hours is not a window") + return (operating_hours * SECONDS_PER_HOUR) / self.human_seconds_per_plate + + def why(self) -> str: + """The finding, including the sentence that makes it a finding rather than a ratio.""" + base = ( + f"a human is needed on {self.intervention_rate:g} of every plate for " + f"{self.minutes_per_intervention:g} min, which is {self.human_seconds_per_plate:.1f} s of " + f"attention per plate against a {self.interval_seconds:.1f} s admission interval: " + f"{self.consumed_fraction * 100:.1f} percent of the clock" + ) + if self.reachable: + tail = ( + f". {self.slack_seconds:.1f} s of each interval is left for everything else, and " + "everything else includes the upkeep and the recovery" + ) + else: + tail = ( + f". At or above 100 percent the queue grows without bound and the target is " + f"unreachable: it needs at least {self.operators_required} operators in parallel, or " + "fewer interventions, or a longer interval. Instrument speed appears nowhere in this " + "arithmetic, so a faster robot does not move it in either direction" + ) + return base + tail + + +def headroom( + interval_seconds: float, + intervention_rate: float, + minutes_per_intervention: float, +) -> Headroom: + """What a human intervention rate costs against a launch interval, and when it kills the target. + + All three inputs come from the caller and none is invented. An intervention rate is a thing + a lab can count in a week without instrumenting anything, which is why this is the one + measurement in this module worth asking for before any duration study. + """ + if interval_seconds <= 0: + raise ValueError(f"{interval_seconds} s is not an admission interval") + if intervention_rate < 0: + raise ValueError("an intervention rate cannot be negative") + if minutes_per_intervention < 0: + raise ValueError("an intervention cannot take negative time") + human = intervention_rate * minutes_per_intervention * 60.0 + return Headroom( + interval_seconds=interval_seconds, + intervention_rate=intervention_rate, + minutes_per_intervention=minutes_per_intervention, + human_seconds_per_plate=human, + consumed_fraction=human / interval_seconds, + ) + + +# -- the honest check ---------------------------------------------------------- + + +class Demonstrated(str, Enum): + """What a published work establishes about RATE, as distinct from capability. + + The line is between a run that HAPPENED and a rate that is HELD. A campaign that ran for + seventeen days is a real and difficult result and it is not a throughput specification: it + is one uncontrolled run, in one domain, with no replication and no baseline. SUSTAINED_RATE + is the category a plates-per-day target actually needs support from, and no row in this + module's table carries it. The member exists so that fact is computed from an empty + selection rather than asserted in a comment. + """ + + NO_RATE_CLAIM = "no_rate_claim" # the work reports no achieved per-day figure at all + CAPABILITY = "capability" # the system did the thing, under observation, once or a few times + CAMPAIGN = "campaign" # it ran continuously for days or longer, within one domain + SUSTAINED_RATE = "sustained_rate" # a production rate, held across domains, over time + + @property + def establishes_a_rate(self) -> bool: + return self is Demonstrated.SUSTAINED_RATE + + +# Full citations, in one place, so a row cannot drift from its reference. +REFERENCES: Dict[str, str] = { + "a_lab": ( + "Szymanski, N. J. et al. 'An autonomous laboratory for the accelerated synthesis of " + "inorganic materials.' Nature 624, 86-91 (2023). doi:10.1038/s41586-023-06734-w. " + "Author Correction: Nature 650, E1 (19 Jan 2026), doi:10.1038/s41586-025-09992-y -- " + "cite the corrected version" + ), + "alabos": ( + "Fei, Y. et al. 'AlabOS: a Python-based reconfigurable workflow management framework for " + "autonomous laboratories.' Digital Discovery 3(11), 2275-2288 (2024). doi:10.1039/D4DD00129J" + ), + "roboculture": ( + "Angers, K. et al. 'RoboCulture enables modular robotic plate-based cell automation.' " + "Cell Reports Physical Science 7(7), 103335 (2026). doi:10.1016/j.xcrp.2026.103335" + ), + "dai": ( + "Dai, T. et al. 'Autonomous mobile robots for exploratory synthetic chemistry.' " + "Nature 635, 890-897 (2024). doi:10.1038/s41586-024-08173-7" + ), + "coley": ( + "Coley, C. W. et al. 'A robotic platform for flow synthesis of organic compounds informed " + "by AI planning.' Science 365(6453), eaax1566 (2019). doi:10.1126/science.aax1566" + ), + "jump": ( + "Chandrasekaran, S. N. et al. 'Three million images and morphological profiles of cells " + "treated with matched chemical and genetic perturbations.' Nature Methods 21(6), 1114-1121 " + "(2024). doi:10.1038/s41592-024-02241-6" + ), + "homecage_multi": ( + "Eswaraka, J. et al. 'Enhanced health evaluation in mice using continuous home-cage " + "monitoring and machine learning: a multicentric study.' Lab Animal 55, 275-281 (2026). " + "doi:10.1038/s41684-026-01745-2" + ), + "smartkage": ( + "Ho, H. et al. 'A fully automated home cage for long-term continuous phenotyping of mouse " + "cognition and behavior.' Cell Reports Methods 3(7), 100532 (2023). " + "doi:10.1016/j.crmeth.2023.100532" + ), + "ncats": ( + "Iversen, P. W. et al. 'HTS Assay Validation.' In: Assay Guidance Manual. Bethesda (MD): " + "Eli Lilly & Company and NCATS, 2012 (rev. 2012 Oct 1). PMID 22553862, Bookshelf NBK83783" + ), + "pylabrobot": ( + "Wierenga, R. P. et al. 'PyLabRobot: An open-source, hardware-agnostic interface for " + "liquid-handling robots and accessories.' Device 1(4), 100111 (2023). " + "doi:10.1016/j.device.2023.100111" + ), + "labsafety": ( + "Zhou, Y. et al. 'Benchmarking large language models on safety risks in scientific " + "laboratories.' Nature Machine Intelligence 8, 20-31 (2026). doi:10.1038/s42256-025-01152-1" + ), + "iso23783": ( + "ISO 23783-1:2022, Automated liquid handling systems -- Part 1: Vocabulary and general " + "requirements; and ISO 23783-3:2022, Part 3: Determination, specification and reporting of " + "volumetric performance. ISO/TC 48" + ), + "fda_di": ( + "U.S. Food and Drug Administration, 'Data Integrity and Compliance With Drug CGMP: " + "Questions and Answers -- Guidance for Industry,' December 2018 (Final, Level 1). " + "Docket FDA-2018-D-3984. Contains nonbinding recommendations" + ), +} + + +@dataclass(frozen=True) +class Demonstration: + """One published system, what it actually ran, and what it does not establish. + + `does_not_establish` is required rather than optional and no row leaves it empty. A + citation table whose caveats are optional degrades into a list of names, and a list of + names is what makes a demonstration look like a benchmark. + + `actuates` separates systems that DO things from systems that WATCH them, and the split is + the finding of this whole table: the only works here that operate continuously at scale + across institutions are observational. + """ + + key: str # indexes REFERENCES + system: str + domain: str + ran_for: str # duration in the authors' own terms, not converted + scale: str # what was actually processed, in the authors' own counts + demonstrated: Demonstrated + actuates: bool + does_not_establish: str + + @property + def citation(self) -> str: + return REFERENCES[self.key] + + @property + def establishes_a_rate(self) -> bool: + return self.demonstrated.establishes_a_rate + + +DEMONSTRATIONS: Tuple[Demonstration, ...] = ( + Demonstration( + key="a_lab", + system="A-Lab: robotic powder prep, four box furnaces, automated XRD, active learning", + domain="solid-state synthesis of inorganic oxide and phosphate powders", + ran_for="17 days of continuous operation", + scale="36 compounds realized from a set of 57 computationally nominated targets", + demonstrated=Demonstrated.CAMPAIGN, + actuates=True, + does_not_establish=( + "no per-day rate is stated and a synthesis target is not a plate. One uncontrolled run " + "on one instrument, with no replication and no human baseline, so 36 of 57 describes " + "that run rather than a capability. Success meant a phase judged realized by automated " + "XRD and Rietveld refinement, not expert review -- the senior author conceded a human " + "could refine these samples better. The 2026 Author Correction removed 'novel' from the " + "title and reclassified four reported successes as inconclusive, and Leeman et al., " + "PRX Energy 3, 011002 (2024) reported substantial errors in the automated XRD analysis" + ), + ), + Demonstration( + key="alabos", + system="AlabOS: task-graph orchestration, resource reservation, sample-position tracking", + domain="inorganic solid-state powder synthesis, one prototype laboratory", + ran_for="approximately 1.5 years of deployment", + scale=( + "around 3,500 samples processed, with a stated peak of 149 samples on one day " + "(9 February 2024)" + ), + demonstrated=Demonstrated.CAMPAIGN, + actuates=True, + does_not_establish=( + "the only per-day figure in this table is a PEAK day, which is the opposite of a " + "sustained cadence -- and dividing 'around 3,500' by 'approximately 1.5 years' would put " + "a precision on both figures that neither carries, so this module does not. The 3,500 is " + "a deployment usage statistic, not successful syntheses or discoveries. Single lab, " + "single domain, no multi-site deployment and no head-to-head benchmark against any other " + "workflow manager. The authors describe the scheduler as first-come-first-serve with no " + "task-duration modeling and state that throughput optimization is future work. Autonomy " + "is human-in-the-loop by design: the user-request module exists because error recovery " + "and consumable replacement need operators" + ), + ), + Demonstration( + key="dai", + system="mobile robots linking a synthesis platform, UPLC-MS, and an 80-MHz benchtop NMR", + domain="exploratory organic and supramolecular synthesis", + ran_for="almost four consecutive days in the diversification campaign; three days in another", + scale="12 analogues via three autonomous synthetic steps; 18 combinations screened", + demonstrated=Demonstrated.CAMPAIGN, + actuates=True, + does_not_establish=( + "no rate is claimed. The authors' speed claim is narrowly that the algorithmic decisions " + "are effectively instantaneous -- the data-inspection step, not synthetic throughput. The " + "decision-maker is a tiered heuristic with human-set thresholds, not machine learning. " + "Most relevant here: an unexpected intramolecular cyclization produced a species with the " + "same molecular weight as the expected product, indistinguishable by UPLC or MS, and it " + "escaped the automated decision rule entirely -- it was caught by human inspection of the " + "NMR and confirmed by single-crystal X-ray diffraction, an off-workflow manual technique. " + "That is an intervention with a rate, and it is the rate `headroom` asks for" + ), + ), + Demonstration( + key="roboculture", + system="general-purpose arm with vision, force feedback, and a behavior-tree framework", + domain="yeast (S. cerevisiae) culture in a 96-well plate", + ran_for="a fully autonomous 15-hour experiment", + scale="one 96-well plate, three replicate groups; 99.0 percent tip insertion over 576 attempts", + demonstrated=Demonstrated.CAPABILITY, + actuates=True, + does_not_establish=( + "the authors explicitly disclaim throughput: they position the platform as operating more " + "like an autonomous vehicle tailored to individual or small-group use than for " + "high-throughput screening, and do not claim to beat commercial liquid handlers. Autonomy " + "is narrower than the headline: initial seeding was manual AND the subculture split was " + "manually initiated as a safety precaution, so the headline decision was human-authorized. " + "Perception is fiducial-tag dependent and the authors call the calibration fragile and " + "time-consuming to maintain. One 15-hour run, one plate, one organism" + ), + ), + Demonstration( + key="coley", + system="reconfigurable continuous-flow platform, robot-arm reconfigured, AI route planning", + domain="flow synthesis of known drug and drug-like substances", + ran_for="not established from the primary source available", + scale="15 drug or drug-like substances", + demonstrated=Demonstrated.NO_RATE_CLAIM, + actuates=True, + does_not_establish=( + "no throughput, cost, or scale comparison against manual synthesis is claimed anywhere in " + "the work. It is not autonomous end to end: the AI proposes routes and an expert chemist " + "must translate each one into an executable recipe file, which the authors attribute to " + "the literature not containing enough information to go from inspiration to execution. " + "Fifteen successes is a demonstration set, not a success rate -- no denominator of " + "attempted-and-failed targets is available -- so it supports no statement about " + "reliability. Widely circulated duration figures for this platform come from press " + "coverage rather than the paper and are deliberately not repeated here" + ), + ), + Demonstration( + key="jump", + system="image-based morphological profiling at plate scale (CPJUMP1)", + domain="compound, CRISPR and ORF perturbations in two cell lines, 384-well format", + ran_for="a single experiment at a single facility", + scale="51 physical plates yielding 107 plates of images; roughly 3 million images", + demonstrated=Demonstrated.NO_RATE_CLAIM, + actuates=True, + does_not_establish=( + "no per-day plate rate is reported and none can be recovered from the paper. The headline " + "finding is negative: the authors conclude that identifying matches between chemical and " + "genetic perturbations is a challenging task, so this is a benchmark resource rather than " + "a solved capability. Single experiment at a single facility, which the authors say " + "limits generalizability; 160 genes, 303 compounds, one concentration the authors call " + "not ideal, two cell lines, two time points" + ), + ), + Demonstration( + key="homecage_multi", + system="capacitance-sensing digital ventilated cages with an ML alerting model", + domain="mouse home-cage locomotion monitoring across three institutions", + ran_for="approximately 300 days at two sites and over 1,000 days at a third", + scale="1,367 cages; 90,540 + 51,094 + 15,367 cage evaluations", + demonstrated=Demonstrated.CAMPAIGN, + actuates=False, + does_not_establish=( + "the largest and longest continuously-operating system in this table performs no " + "experiment. It observes. It executes nothing, moves nothing, and decides nothing at the " + "bench, so it establishes no throughput of any kind. The analysis is retrospective, with " + "no prospective or interventional validation that earlier alerts improved any outcome. " + "Alerts are not real-time -- the authors state they are not reported to caretakers until " + "the morning. Resolution is cage-level, not animal-level; the algorithm missed 9-20 " + "percent of verified clinical cases; and four co-authors are employed by the vendor of " + "the sensing platform being evaluated" + ), + ), + Demonstration( + key="smartkage", + system="automated home cage running T-maze, novel object and object-in-place tasks", + domain="mouse cognitive and behavioral phenotyping, no food or water restriction", + ran_for="continuous testing over approximately 4.5 months in the longest arm", + scale="one mouse per cage; lesion groups of n=5 and n=4, AD arm of 5 knock-in mice", + demonstrated=Demonstrated.CAMPAIGN, + actuates=True, + does_not_establish=( + "the authors call their own system low throughput, in those words, and note the " + "single-mouse design is suboptimal for stress as well. It runs one animal at a time, so " + "months of continuous unattended operation produce one subject's data. Underpowered by " + "the authors' own limitations section, single animal facility, no multi-site replication. " + "The medial-entorhinal arm classified 1 of 4 correctly, so the discrimination claim holds " + "for hippocampal lesions and barely at all for the other lesion type. The tracking " + "training data is proprietary and cannot be released, so the pipeline is not reproducible " + "end to end" + ), + ), + Demonstration( + key="ncats", + system="pre-production HTS assay validation designs (plate uniformity, replicate experiment)", + domain="biochemical and cell-based screening assays in drug discovery", + ran_for="a 3-day plate-uniformity study", + scale=( + "3 plates per day in the interleaved-signal format, or 6 per day in the CRC plus " + "uniform-signal format" + ), + demonstrated=Demonstrated.NO_RATE_CLAIM, + actuates=False, + does_not_establish=( + "the plates-per-day figures here are a prescribed STUDY DESIGN, not an achieved rate, and " + "quoting 3 or 6 as a demonstrated throughput would invert what the chapter is for. It is " + "pre-production readiness guidance: the authors state the replicate-experiment study is a " + "decision tool to establish that an assay is ready to go into production and is not a " + "substitute for post-production monitoring. It is an industry and NIH best-practice " + "manual, not GLP, GMP, CLIA or FDA guidance, and its Z' >= 0.4 criterion is specific to " + "this context rather than a universal quality bar" + ), + ), + Demonstration( + key="pylabrobot", + system="hardware-agnostic Python interface over three liquid-handler families plus readers", + domain="liquid-handling control software", + ran_for="not applicable; this is a software interface rather than a run", + scale="three robot families (liquid handler/STARLet, Tecan Freedom EVO, Opentrons OT-2)", + demonstrated=Demonstrated.NO_RATE_CLAIM, + actuates=True, + does_not_establish=( + "the paper reports no throughput or timing benchmark of any kind, and no pipetting " + "accuracy or precision data either -- any performance number attributed to it would be " + "fabricated. Testing stops at command generation: unit tests check that user input " + "produces consistent machine-level commands, not that the liquid arrives. Backends are " + "not at parity, and the simulator claim is architectural -- it is designed to work with " + "any robot for which a deck model exists, and per-robot work is still required" + ), + ), +) + + +DEMONSTRATIONS_BY_KEY: Dict[str, Demonstration] = {d.key: d for d in DEMONSTRATIONS} + + +NO_INTEGRATED_PRODUCTION_RATE = ( + "No cited published system establishes integrated multi-domain autonomous laboratory " + "operation at a production rate. The longest integrated campaign in the cited record is 17 " + "days in a single domain on inorganic powders; the only per-day figure is a peak day from a " + "single-lab deployment whose authors call its scheduler non-optimal; and the one system that " + "runs continuously at scale across institutions is a cage sensor that observes rather than " + "actuates, retrospectively, with its alerts held until morning. A plates-per-day target is " + "therefore an engineering commitment, not a demonstrated capability, and nothing in this " + "module will return a citation that supports one" +) + + +@dataclass(frozen=True) +class Support: + """Whether the published record supports a target, and what the nearest evidence actually is. + + `supported` is a property computed from the table rather than a stored False. If someone + later adds a genuine SUSTAINED_RATE row, this flips on its own, which is the only honest way + to write a claim that is currently negative -- a hard-coded False would stop being computed + the moment it stopped being true. + """ + + target: Target + refused: bool + reason: str + nearest: Tuple[Demonstration, ...] = () + + def supporting(self) -> Tuple[Demonstration, ...]: + return tuple(d for d in self.nearest if d.establishes_a_rate) + + @property + def supported(self) -> bool: + return bool(self.supporting()) + + +def demonstrated_support(plates_per_day) -> Support: + """Has anybody published a system that sustains this rate? + + Takes a `Target` or a bare number. A bare number is refused, because a number alone does not + say whether it counts plates started, completed, QC-passed, or released, and the four differ + by the failure and rerun rate. This matters more here than in `cadence_for`: comparing a + target against the published record without knowing which endpoint it names would compare + two different quantities and call the mismatch evidence. + + When the endpoint IS stated, the answer is still no. What the function returns instead is + every nearest work with what it does not establish, so the refusal is checkable rather than + merely asserted. + """ + target = ( + plates_per_day + if isinstance(plates_per_day, Target) + else Target(plates_per_day=float(plates_per_day)) + ) + if target.ambiguous: + return Support( + target=target, + refused=True, + reason=( + target.why_ambiguous() + + ". Until it does, there is nothing to check the published record against, and this " + "function returns no citations rather than the ones that would look most supportive" + ), + nearest=(), + ) + return Support( + target=target, + refused=False, + reason=( + f"{NO_INTEGRATED_PRODUCTION_RATE}. The target of {target.plates_per_day:g} plates " + f"{target.endpoint.value} per day is unsupported at any value; the {len(DEMONSTRATIONS)} " + "works below are the nearest published evidence and each is listed with what it does not " + "establish" + ), + nearest=DEMONSTRATIONS, + ) + + +def sustained_rate_demonstrations() -> Tuple[Demonstration, ...]: + """Every cited work that establishes a sustained production rate. Currently none. + + Computed from the table rather than stated, so the emptiness is a result. The function + exists so a caller can check the claim instead of taking the module docstring's word for it. + """ + return tuple(d for d in DEMONSTRATIONS if d.establishes_a_rate) + + +__all__ = [ + "DEMONSTRATIONS", + "DEMONSTRATIONS_BY_KEY", + "NO_INTEGRATED_PRODUCTION_RATE", + "REFERENCES", + "SECONDS_PER_DAY", + "SECONDS_PER_HOUR", + "Cadence", + "Demonstrated", + "Demonstration", + "Endpoint", + "Forcing", + "Headroom", + "Implication", + "Support", + "Target", + "cadence_for", + "demonstrated_support", + "headroom", + "intervals", + "launch_interval", + "required_admissions", + "sustained_rate_demonstrations", +] diff --git a/autonomous_lab/cli.py b/autonomous_lab/cli.py index 16c258a..e716a75 100644 --- a/autonomous_lab/cli.py +++ b/autonomous_lab/cli.py @@ -14,6 +14,7 @@ import argparse import logging import sys +import textwrap from typing import List from .doctor import ( @@ -31,7 +32,10 @@ loop_closure, untrusted_ops, ) +from .authority import DECISION_CLASSES, LADDER, Authority +from .cadence import Endpoint, Target, cadence_for, demonstrated_support, headroom from .ledger import build_ledger, rank_unlocks +from .literature import EVIDENCE, Confidence, summary as literature_summary, unsupported_claims from .lineage import MISASSIGNMENT, Separability, lineage_report from .model import Verdict from .provenance import provenance_report @@ -380,6 +384,117 @@ def _lineage(args) -> int: return 0 if report.verdict.ok else 1 +def _evidence(args) -> int: + """What the published record actually establishes -- and what it does not.""" + s = literature_summary() + print("published evidence for laboratory autonomy\n") + print(f" works cited {s['evidence']} ({s['confirmed']} confirmed, {s['partial']} partial)") + print(f" carrying unverified numbers {s['entries_with_unverified_numbers']}") + print(f" domains covered {s['domains_covered']}\n") + + for e in EVIDENCE: + if args.domain and e.domain.value != args.domain: + continue + mark = "" if e.confidence is Confidence.CONFIRMED else " [partially verified]" + print(f" {e.key} ({e.domain.value}, {e.year}){mark}") + print(f" shows {(e.shorthand or e.demonstrates)[:160]}") + print(f" DOES NOT {e.does_not_establish[:200]}") + if e.unverified: + print(f" unverified {len(e.unverified)} figure(s) could not be confirmed") + print(f" {e.url}") + print() + + claims = unsupported_claims() + print(f" {len(claims)} claim(s) NO cited work establishes:\n") + for c in claims: + print(f" {c.name}") + print(f" {c.claim}") + print(f" would establish it: {c.would_establish_it}") + print() + print( + " A paper supports a claim by its SCOPE, not by its fame. A seventeen-day campaign\n" + " and a fifteen-hour single-plate run are real results, and neither is a throughput\n" + " specification." + ) + return 0 + + +def _authority(args) -> int: + """Which decisions a model may make, and which need an interlock or a person.""" + print("decision authority\n") + for level in (Authority.MODEL, Authority.MODEL_PROPOSES, Authority.DETERMINISTIC, Authority.HUMAN): + rows = [d for d in DECISION_CLASSES if d.required is level] + if not rows: + continue + print(f" {level.value.upper()}") + for d in rows: + print(f" {d.name}") + print(f" decides {d.decides}") + print(f" because {d.because}") + if d.evidence_key: + print(f" evidence {d.evidence_key}") + print() + + print(" qualification ladder -- no rung implies the one above it:\n") + for r in LADDER: + print(f" {r.rung.value}") + print(f" qualifies {r.qualifies}") + print(f" does NOT {r.does_not_qualify}") + print( + "\n Authority may be strengthened and never weakened. Putting a person on a decision\n" + " a model could make is a cost; the reverse is the defect this exists to catch." + ) + return 0 + + +def _cadence(args) -> int: + """What a plates-per-day target actually costs, before anything is bought.""" + if args.endpoint is None: + t = Target(plates_per_day=args.plates_per_day) + print(f"cadence for {args.plates_per_day:g} plates/day\n") + print(" REFUSED") + print(f" {t.why_ambiguous()}") + print("\n Re-run with --endpoint started|completed|qc-passed|released.") + return 1 + + target = Target( + plates_per_day=args.plates_per_day, + endpoint=Endpoint(args.endpoint.replace("-", "_")), + ) + c = cadence_for(target, operating_hours=args.hours, slowest_step_seconds=args.slowest_step) + print(f"cadence for {target.plates_per_day:g} plates {target.endpoint.value}/day\n") + print( + f" admission interval {c.interval_seconds:.1f} s " + f"({c.minutes_between_plates:.2f} min) over {c.operating_hours:g} h" + ) + print(f" round the clock {c.round_the_clock_seconds:.1f} s") + print(f" walk-away penalty {c.walk_away_multiplier:.2f}x for not running unattended\n") + + for imp in c.implied_constraints(): + flag = " <-- UNACHIEVABLE" if imp.unachievable else "" + print(f" {imp.forcing.value.upper()}{flag}") + for line in textwrap.wrap(imp.statement, 84): + print(f" {line}") + if imp.arithmetic: + print(f" = {imp.arithmetic}") + print() + + if args.intervention_rate is not None: + h = headroom(c.interval_seconds, args.intervention_rate, args.minutes_per_intervention) + print( + f" HUMAN SHARE {h.human_seconds_per_plate:.1f} s per plate = " + f"{h.consumed_fraction:.0%} of the interval" + ) + if h.consumed_fraction >= 1.0: + print(" The target is unreachable at this intervention rate. Instrument speed is") + print(" not in this arithmetic; only closing the steps that need a human is.") + print() + + print(" PUBLISHED SUPPORT") + print(f" {demonstrated_support(target).reason}") + return 0 + + def _knowledge(args) -> int: """The tacit layer: expert judgments and the benchmarks a robot must meet.""" s = knowledge_summary() @@ -523,6 +638,25 @@ def common(sp): ln.add_argument("--unit", default="cell", help="what the science needs to distinguish") ln.set_defaults(func=_lineage) + ev = sub.add_parser("evidence", help="what the published record establishes, and what it does not") + ev.add_argument("--domain", help="filter to one domain") + ev.set_defaults(func=_evidence) + + au = sub.add_parser("authority", help="which decisions a model may make, and which need a person") + au.set_defaults(func=_authority) + + cd = sub.add_parser("cadence", help="what a plates-per-day target actually costs") + cd.add_argument("plates_per_day", type=float) + cd.add_argument("--endpoint", choices=["started", "completed", "qc-passed", "released"], + help="what the target counts; refused without it") + cd.add_argument("--hours", type=float, default=24.0) + cd.add_argument("--slowest-step", dest="slowest_step", type=float, + help="seconds of the slowest single step, if measured") + cd.add_argument("--intervention-rate", dest="intervention_rate", type=float, + help="human interventions per plate") + cd.add_argument("--minutes-per-intervention", dest="minutes_per_intervention", type=float, default=5.0) + cd.set_defaults(func=_cadence) + kn = sub.add_parser("knowledge", help="encoded expert judgment and robot benchmarks") common(kn) kn.set_defaults(func=_knowledge) diff --git a/autonomous_lab/literature.py b/autonomous_lab/literature.py new file mode 100644 index 0000000..1a09f17 --- /dev/null +++ b/autonomous_lab/literature.py @@ -0,0 +1,1495 @@ +"""What the published field has actually demonstrated, and the gap to what is being claimed. + +Every other module in this package answers a question about THIS lab: which steps run +headless, which gates can fire, what a camera could see, what a run could prove afterwards. +None of them can check an ambition against the outside world. That is the gap this module +closes, and it exists because of a specific, routine, and expensive failure of reasoning. + +An autonomous-lab claim is almost never defended on its own evidence. It is defended by +pointing at a famous paper. The paper is real, the result is real, and the scope of the +result is far narrower than the claim it is being made to carry -- a solid-state materials +campaign gets cited to justify a cell-biology production line, a fifteen-hour single-plate +yeast run gets cited to justify sustained unattended culture, a text benchmark gets cited to +justify letting a model judge hazards at a bench. Nobody in that chain lies. The citation is +accurate and the inference is not, and there is no artifact anywhere in the usual workflow +that makes the inference checkable. + +So each published work here carries two fields, and the second one is the reason the module +exists. `demonstrates` records what the work showed, with the real numbers. `does_not_establish` +records the boundary its own authors, correctors, or published critics drew around it. A +citation with no stated scope limit is not a strong citation; it is an unread one, and the +tests refuse to let one exist in this table. + +The load-bearing computation is `support_for(claim_domain, scope)`. Given a domain and a +claimed operating scope it returns the evidence that speaks to the claim at all, and for each +one the SHORTFALL between what was demonstrated and what is being asserted. Domain is the +first cut and it is harsh on purpose: inorganic powders do not contaminate, do not lose +viability, and do not drift in state between Tuesday and Thursday, so a materials campaign is +not weak evidence about a cell assay, it is evidence about something else. Duration and +repetition are the second cut, and they are where the in-domain citations fall down. Nothing +in this table demonstrates a lab operating without a human at a production rate, and +`unsupported_claims()` is the standing list of the things that keep getting asserted anyway. + +CONFIDENCE mirrors `qc.Basis`. There, a threshold from a kit insert and a threshold this lab +titrated are both real and only one is validated. Here, an entry whose every stated number was +confirmed against the source and an entry where an attribution could not be confirmed are both +real and only one is CONFIRMED. The unconfirmable numbers are carried explicitly rather than +dropped or rounded away, because a verification pass whose misses are invisible is worth less +than no verification pass -- it produces the same table with unearned authority. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Dict, List, Optional, Tuple + + +class Domain(str, Enum): + """The field a published work actually operated in. + + The load-bearing line is LIVING. Material that is alive fails in ways non-living matter + cannot: it contaminates, it loses viability, it changes state while it waits, and it is + never twice in the same condition. A demonstration on inorganic powder or on small + molecules in flow says nothing about any of those, however impressive the automation, so + cross-domain citation is not a matter of degree here. It is a category error, and drawing + the line in the enum is what makes it computable rather than a matter of taste. + """ + + MATERIALS = "materials" # solid-state inorganic synthesis and characterization + CHEMISTRY = "chemistry" # organic and supramolecular synthesis + CELL_ASSAY = "cell_assay" # cultured cells, plate-based + IN_VIVO = "in_vivo" # whole animals + ORCHESTRATION = "orchestration" # workflow management across instruments + PROVENANCE = "provenance" # records, custody, audit + SAFETY = "safety" # hazard knowledge and hazard judgment + ABSTRACTION = "abstraction" # hardware-agnostic control interfaces and their standards + + @property + def living(self) -> bool: + return self in (Domain.CELL_ASSAY, Domain.IN_VIVO) + + +class EvidenceKind(str, Enum): + """What kind of thing a published work is, which decides what it can ever support. + + The line is DEMONSTRATES_OPERATION, and only EMPIRICAL_DEMONSTRATION crosses it. The other + four are routinely cited as though they did. A BENCHMARK measures models or methods against + constructed tasks and no lab ran. A RESOURCE supplies data or software and no scientific + question was answered. GUIDANCE and STANDARD say what ought to be done and are evidence that + a practice is recommended, never that anyone achieved it. Citing a standard as proof of + capability is the most common form of this error, because a standard's existence is easy to + mistake for its satisfaction. + """ + + EMPIRICAL_DEMONSTRATION = "empirical_demonstration" # a lab ran and reported what happened + BENCHMARK = "benchmark" # constructed tasks scoring models or methods + GUIDANCE = "guidance" # recommended practice, nonbinding + STANDARD = "standard" # a normative specification + RESOURCE = "resource" # a dataset or a piece of software + + @property + def demonstrates_operation(self) -> bool: + return self is EvidenceKind.EMPIRICAL_DEMONSTRATION + + +class Confidence(str, Enum): + """How much of what this module states about a work was confirmed against the source. + + The same distinction `qc.Basis` draws between a vendor's number and this lab's own. PARTIAL + is not a doubt about the work; the works here are real and correctly cited. It means a + number or an attribution that this module would otherwise state could not be checked at the + source, usually because the article is paywalled and verification rested on a preprint or on + press coverage. Those items are listed in `unverified` rather than quietly softened, because + the alternative is a table where the checked and the unchecked look identical. + """ + + PARTIAL = "partial" # at least one stated number or attribution is unconfirmed + CONFIRMED = "confirmed" # every number stated in `demonstrates` was confirmed at the source + + @property + def fully_verified(self) -> bool: + return self is Confidence.CONFIRMED + + +@dataclass(frozen=True) +class Scope: + """An operating envelope: either what a work demonstrated, or what a claim asserts. + + One type for both sides on purpose. A scope gap is then a comparison of like with like + rather than prose judgment, and a claim that cannot be written as a scope is a claim too + vague to check -- which is itself worth discovering before it reaches a slide. + + `continuous_hours` exists only to make duration comparable and is None wherever the source + hedges. Dai et al. wrote "almost four consecutive days"; converting that to 96 would + manufacture a precision the authors deliberately withheld. `duration_verbatim` carries the + authors' own words and is the field to quote. + """ + + summary: str + material: str = "" + duration_verbatim: str = "" + continuous_hours: Optional[float] = None + # One campaign is descriptive of that campaign. Without repetition a success rate is a + # record of what happened once, not a capability. + replicated: bool = False + # Repeated campaigns at a steady rate over a deployment, as opposed to a single bounded + # run. This is the field every production claim turns on and no demonstration here sets. + sustained: bool = False + # Asserted only by claims. A demonstration declares `human_in_loop` instead. + unattended: bool = False + # What a person still did. Empty on a demonstration means fully unattended operation, and + # nothing in this table is entitled to leave it empty. + human_in_loop: str = "" + + +@dataclass(frozen=True) +class Evidence: + """One published work, what it showed, and the boundary around it. + + `shorthand` is the phrase this module uses when it refuses a citation, and it is built from + verified numbers only -- "a 17-day materials campaign", "a 15-hour single-plate yeast run". + Naming the scope in the refusal is the whole point: "that paper does not support this" is an + opinion, and "a 15-hour single-plate yeast run does not reach sustained production" is a + statement someone can check and argue with. + + `does_not_establish` is never optional and never generic. It carries what the authors, the + published corrections, or the published critics said the work does not show. + """ + + key: str + citation: str + url: str + domain: Domain + kind: EvidenceKind + year: int + confidence: Confidence + shorthand: str + demonstrates: str + does_not_establish: str + scope: Scope + # Specific numbers and attributions that could not be confirmed at the source. Carried on + # CONFIRMED entries too: those are figures in circulation that this module declines to + # state, and recording them is what stops a later editor from reintroducing one. + unverified: Tuple[str, ...] = () + + def __post_init__(self) -> None: + if not self.does_not_establish.strip(): + raise ValueError( + f"evidence '{self.key}' states no scope limit; a citation whose boundary nobody " + "wrote down is the failure mode this module exists to prevent" + ) + if not self.url.strip(): + raise ValueError(f"evidence '{self.key}' carries no URL or DOI; it cannot be checked") + if self.confidence is Confidence.PARTIAL and not self.unverified: + raise ValueError( + f"evidence '{self.key}' is PARTIAL but names nothing unverified; a confidence " + "downgrade with no reason attached cannot be acted on or retired" + ) + + @property + def verified(self) -> bool: + return self.confidence.fully_verified + + +# -- the works themselves ------------------------------------------------------- +# Numbers are the ones confirmed at the source. Where a figure is in circulation and did not +# survive checking, it is named in `unverified` and stated nowhere else. + +EVIDENCE: Tuple[Evidence, ...] = ( + Evidence( + key="a_lab", + citation=( + "Szymanski, N. J. et al. 'An autonomous laboratory for the accelerated synthesis of " + "inorganic materials.' Nature 624, 86-91 (2023). doi:10.1038/s41586-023-06734-w. " + "Author Correction: Nature 650, E1 (19 January 2026), doi:10.1038/s41586-025-09992-y; " + "the corrected version is the one cited here" + ), + url="https://doi.org/10.1038/s41586-023-06734-w", + domain=Domain.MATERIALS, + kind=EvidenceKind.EMPIRICAL_DEMONSTRATION, + year=2023, + confidence=Confidence.CONFIRMED, + shorthand="a 17-day materials campaign", + demonstrates=( + "Autonomous solid-state synthesis of computationally nominated inorganic targets. Over " + "17 days of continuous operation the platform realized 36 compounds from a set of 57 " + "targets, a 63 percent rate; the remainder were 4 reclassified as inconclusive and 17 " + "outright failures. Robotic powder dispensing and mixing, a UR5e arm on a linear rail, " + "four box furnaces, automated X-ray diffraction with phase identification against the " + "Inorganic Crystal Structure Database followed by automated Rietveld refinement. Routes " + "were improved in the loop by the ARROWS3 algorithm from pairwise reactions observed in " + "failed attempts; initial recipes came from natural-language models trained on the " + "literature. Targets were oxides and phosphates drawn from ab initio phase-stability data" + ), + does_not_establish=( + "Not discovery of new-to-science materials. The 2026 Author Correction removed 'novel' " + "from the title and the authors clarified that the intended meaning was new to the " + "prediction platform, not new to science; the paper establishes autonomous synthesis and " + "realization of nominated targets. The headline figure is itself a downgrade from the " + "original 41 of 58: one target was removed as present in the algorithms' training data " + "and so not a blind target, and 4 of the remaining reported successes were reclassified " + "as inconclusive from XRD alone -- any pre-2026 copy still shows 41/58 and the old " + "title. Substantive external critique remains open: Leeman et al., PRX Energy 3, 011002 " + "(2024) reported substantial errors in the automated powder-XRD analysis and concluded " + "there was insufficient evidence for the discovery of new materials, the core objection " + "being that the pipeline did not handle compositional disorder, so ordered versions of " + "already-known disordered compounds were read as new phases. The correction acknowledges " + "concerns about unambiguous identification by diffraction without conceding all points. " + "The senior author's public response concedes the analysis limit directly: a human can " + "perform a higher-quality refinement on these samples, and replacing the human " + "materials-discovery process was never the intention. Success means a phase was judged " + "realized by automated XRD and Rietveld refinement -- not single-crystal confirmation, " + "not independent replication, not expert review. No material property was measured, no " + "application was demonstrated, and no claim is made about scale-up or about the value of " + "any compound produced. One campaign, one instrument, no baseline against human chemists " + "and no repetition, so 63 percent describes this run and is not a capability benchmark" + ), + scope=Scope( + summary="one continuous campaign of solid-state synthesis on inorganic oxide and phosphate powders", + material="inorganic oxide and phosphate powders", + duration_verbatim="over 17 days of continuous operation", + continuous_hours=408.0, + replicated=False, + sustained=False, + human_in_loop=( + "the four reclassified targets and the disorder objection were both resolved by human " + "reading of the diffraction data, not by the workflow" + ), + ), + unverified=( + "the per-failure-mode breakdown does not reconcile: sluggish kinetics 11, precursor " + "volatility 1, amorphization 1, computational inaccuracy 3 sums to 16 against 17 failed " + "targets, so no per-mode count is stated here", + "which 4 compounds were reclassified as inconclusive; the corrected figure and revised " + "supplementary data could not be retrieved", + "whether any of the 36 successes have since been independently reproduced by a non-XRD " + "method; no evidence was found either way", + "'43 new materials', which appears in news coverage, matches no version of the paper " + "(41 original, 36 corrected) and is not used", + ), + ), + Evidence( + key="dai", + citation=( + "Dai, T., Vijayakrishnan, S., Szczypinski, F. T., Ayme, J.-F., Simaei, E., Fellowes, T., " + "Clowes, R., Kotopanov, L., Shields, C. E., Zhou, Z., Ward, J. W. & Cooper, A. I. " + "'Autonomous mobile robots for exploratory synthetic chemistry.' Nature 635, 890-897 " + "(2024). doi:10.1038/s41586-024-08173-7" + ), + url="https://www.nature.com/articles/s41586-024-08173-7", + domain=Domain.CHEMISTRY, + kind=EvidenceKind.EMPIRICAL_DEMONSTRATION, + year=2024, + confidence=Confidence.CONFIRMED, + shorthand="an almost-four-day exploratory chemistry campaign", + demonstrates=( + "Mobile robots linking an automated synthesis platform to remotely located, unmodified " + "UPLC-MS and 80-MHz benchtop NMR instruments. The structural-diversification campaign " + "operated for almost four consecutive days under autonomous decision making; a separate " + "supramolecular campaign ran continuously over 3 days. Three campaigns in total -- " + "structural diversification, supramolecular host-guest, photochemical synthesis -- " + "producing 12 structurally diverse analogues through 3 autonomous synthetic steps from 6 " + "targeted ureas and thioureas, screening 18 possible combinations, and testing six " + "photocatalysts against a blank control. Two task-specific mobile robots were used, and " + "the same tasks were also demonstrated with a single robot and a multipurpose gripper" + ), + does_not_establish=( + "Not autonomous structure elucidation and not autonomous anomaly detection, and the " + "paper supplies its own counterexample. Compound 13 underwent an unexpected " + "intramolecular cyclization to a species with the same molecular weight as the expected " + "product, indistinguishable by UPLC chromatogram or MS alone. The workflow acquired the " + "NMR data; the heuristic decision rule did not flag it, and it was identified by human " + "inspection and confirmed by single-crystal X-ray diffraction, an off-workflow manual " + "measurement. The authors concede the discoveries required post-experiment analysis by " + "human researchers. It involves no machine learning: the decision maker is an explicit " + "tiered heuristic with success criteria designed by domain experts and thresholds set at " + "the start, positioned by the authors as an alternative to black-box models for " + "analytically complex, data-sparse problems, so it is not evidence about AI-driven or " + "self-improving labs. Analytical resolution is explicitly limited: the low field " + "strength of the 80-MHz benchtop instrument can artificially increase the apparent number " + "of peaks. The speed claim is narrow -- the authors claim only that the algorithmic " + "decisions are effectively instantaneous, which is acceleration of the decision step, not " + "superior synthetic throughput or chemical judgment. Scalability to industrial labs and " + "transfer to other reaction classes are stated as expectations, not results, and the " + "reaction and product space was human-specified up front" + ), + scope=Scope( + summary="three bounded exploratory campaigns within a human-specified reagent space", + material="organic and supramolecular small molecules in solution", + duration_verbatim="almost four consecutive days", + # The authors hedged the figure. Converting "almost four days" to 96 would manufacture + # a precision they withheld, so duration is not comparable for this entry and the gap + # says so rather than guessing. + continuous_hours=None, + replicated=False, + sustained=False, + human_in_loop=( + "success criteria and thresholds were set by domain experts before the run, and the " + "one anomaly that mattered was caught by human inspection plus off-workflow " + "single-crystal X-ray diffraction" + ), + ), + ), + Evidence( + key="coley", + citation=( + "Coley, C. W.; Thomas, D. A., III; Lummiss, J. A. M.; Jaworski, J. N.; Breen, C. P.; " + "Schultz, V.; Hart, T.; Fishman, J. S.; Rogers, L.; Gao, H.; Hicklin, R. W.; Plehiers, " + "P. P.; Byington, J.; Piotti, J. S.; Green, W. H.; Hart, A. J.; Jamison, T. F.; Jensen, " + "K. F. 'A robotic platform for flow synthesis of organic compounds informed by AI " + "planning.' Science 2019, 365 (6453), eaax1566. doi:10.1126/science.aax1566" + ), + url="https://www.science.org/doi/10.1126/science.aax1566", + domain=Domain.CHEMISTRY, + kind=EvidenceKind.EMPIRICAL_DEMONSTRATION, + year=2019, + confidence=Confidence.PARTIAL, + shorthand="a 15-compound flow-synthesis demonstration", + demonstrates=( + "A modular continuous-flow platform, automatically reconfigured by a robotic arm to set " + "up the required unit operations, executing routes proposed by generalization over " + "millions of published reactions and validated in silico. Demonstrated for 15 drug or " + "drug-like substances. Additional implementation details are determined by expert " + "chemists and recorded in reusable recipe files; solvent choice and precise stoichiometry " + "are named as part of that human step" + ), + does_not_establish=( + "Not autonomous or end-to-end AI-driven synthesis. The AI proposes routes only; a human " + "expert chemist must translate each proposed route into an executable recipe file, " + "because the chemical literature does not contain enough information to get from " + "inspiration to execution automatically. The authors' own framing is a step toward a " + "paradigm, computer-augmented. Not discovery: all 15 targets are known drug or drug-like " + "substances made by literature-precedented chemistry, with no novel molecules, no novel " + "reactions, and no medicinal-chemistry findings. Not closed-loop optimization or " + "self-correction -- the platform executes a fixed human-refined recipe, and automated " + "optimization on this same platform arrived only in later work. Not general across " + "chemistry: the system is bounded by what the modular flow hardware can host. Fifteen " + "successful targets is a demonstration set and not a success rate; no verifiable " + "denominator of attempted-and-failed targets exists, so the work supports no statement " + "about reliability" + ), + scope=Scope( + summary="15 known drug or drug-like targets, each executed from a human-authored recipe file", + material="organic small molecules in continuous flow", + duration_verbatim="not stated in any source that could be verified", + continuous_hours=None, + replicated=False, + sustained=False, + human_in_loop=( + "an expert chemist authors the recipe file for every route, supplying solvent and " + "stoichiometry that the route planner does not" + ), + ), + unverified=( + "that safety or hazard assessment is part of the expert refinement step; the abstract " + "says only 'additional implementation details' and no accessible source attributes " + "safety to it, so safety is not claimed here", + "duration figures of roughly two hours for the simplest target and about 68 hours for " + "manufacturing multiple compounds, which appear only in press coverage and are stated " + "nowhere in this entry", + "a breadth figure of about 30 different reactions, also press-only", + "'eight distinct retrosynthetic routes and nine process configurations' returned by a " + "metadata summarizer, which could not be corroborated against the paper and is treated " + "as fabricated", + "the identities of the 15 compounds, which are inconsistent across press coverage and " + "are therefore not enumerated", + ), + ), + Evidence( + key="roboculture", + citation=( + "Angers, K., Darvish, K., Yoshikawa, N., Okhovatian, S., Bannerman, D., Yakavets, I., " + "Shkurti, F., Aspuru-Guzik, A., & Radisic, M. (2026). 'RoboCulture enables modular " + "robotic plate-based cell automation.' Cell Reports Physical Science, 7(7), 103335. " + "doi:10.1016/j.xcrp.2026.103335. Preprint: arXiv:2505.14941" + ), + url="https://doi.org/10.1016/j.xcrp.2026.103335", + domain=Domain.CELL_ASSAY, + kind=EvidenceKind.EMPIRICAL_DEMONSTRATION, + year=2026, + confidence=Confidence.CONFIRMED, + shorthand="a 15-hour single-plate yeast run", + demonstrates=( + "A general-purpose robotic manipulator automating plate-based cell culture using vision " + "and force feedback and a modular behavior-tree framework to execute, monitor, and manage " + "experiments. A fully autonomous 15-hour yeast culture experiment, with visual servoing " + "centering the pipette tip over any well of an arbitrarily positioned 96-well plate: 99.0 " + "percent pipette-tip insertion success over 576 attempts across six random plate " + "placements, 92.4 percent of them perfect. Three replicate groups at initial " + "concentrations of 50, 30 and 10 million cells/mL plus blank controls, with 3x dilution " + "into fresh media at subculturing" + ), + does_not_establish=( + "Not unattended autonomy, and seeding was not the only manual step. The authors state " + "that splitting was manually initiated as a safety precaution: the robot detected " + "saturation and tracked growth, but a human authorized the subculturing decision, which " + "is the headline autonomous-decision capability. Initial seeding of cells likewise " + "remains a manual step performed before the run. The visual servoing does not generalize " + "-- it is specifically tailored for well plates and does not generalize to other " + "laboratory equipment. Perception is fiducial-tag dependent and brittle: it assumes tags " + "remain unoccluded and requires a calibration process that the authors call fragile and " + "time-consuming to maintain. The authors explicitly disclaim generality, describing the " + "system as not yet a fully generalizable system. It is not a throughput claim: they " + "position it as operating more like an autonomous vehicle tailored to individual or " + "small-group use than for high-throughput screening, and it neither beats commercial " + "liquid handlers on throughput nor claims to. The biology is yeast culture only, so it " + "establishes nothing about mammalian or primary cell culture, nothing about sterility " + "over multi-day runs, and no scientific result -- the contribution is robotics. Three " + "replicate groups in one 15-hour run on a single plate is a capability demonstration, " + "not a validated assay" + ), + scope=Scope( + summary="one 15-hour run on one 96-well plate of yeast, with human-initiated splitting", + material="Saccharomyces cerevisiae culture in a 96-well plate", + duration_verbatim="a fully autonomous 15-hour yeast culture experiment", + continuous_hours=15.0, + replicated=False, + sustained=False, + human_in_loop=( + "cell seeding was performed manually before the run, and the subculturing split was " + "manually initiated as a safety precaution" + ), + ), + unverified=( + "every number here was checked against arXiv v2 rather than the version of record; " + "sciencedirect.com and cell.com both returned HTTP 403, and the journal retitled the " + "paper, so the manuscript was revised during review and figures could have shifted", + "the published abstract is not exposed by Crossref or Semantic Scholar, so the 15-hour, " + "96-well and 99.0 percent figures were not re-confirmed in the version of record", + ), + ), + Evidence( + key="jump", + citation=( + "Chandrasekaran, S. N. et al. (2024). 'Three million images and morphological profiles " + "of cells treated with matched chemical and genetic perturbations.' Nature Methods, " + "21(6), 1114-1121. doi:10.1038/s41592-024-02241-6" + ), + url="https://www.nature.com/articles/s41592-024-02241-6", + domain=Domain.CELL_ASSAY, + kind=EvidenceKind.BENCHMARK, + year=2024, + confidence=Confidence.CONFIRMED, + shorthand="the CPJUMP1 image-profiling resource", + demonstrates=( + "A public resource for evaluating chemical-genetic matching by morphological profiling: " + "roughly three million images of cells and image-based profiles of seventy five million " + "single cells, plus well-level aggregated profiles. Compounds, CRISPR knockout guides and " + "ORF overexpression run side by side across 160 genes and 303 compounds, each perturbed " + "gene's product being a known target of at least two compounds in the set. Two cell lines " + "(U2OS, A549), two time points per modality, two guides per gene for all but 15 genes, " + "one ORF per gene, 51 physical plates yielding 107 plates of images in 384-well format at " + "a single 5 uM compound concentration" + ), + does_not_establish=( + "It does not establish that chemical-genetic matching works, and citing it as evidence " + "that morphological profiling successfully links compounds to gene targets inverts the " + "paper's own conclusion, which is that identifying matches between chemical and genetic " + "perturbations is a challenging task. The dataset exists so methods that attempt it can " + "be evaluated. Coverage is narrow at 160 genes and 303 compounds, and about 21 percent of " + "compounds are annotated against more than one gene family -- the authors flag " + "polypharmacology as likely to substantially impair matching and note target annotations " + "may be incomplete. Selection is biased: compounds and genes were curated from a drug " + "repurposing library and a genetic-perturbation library, so the set holds preclinical and " + "clinical compounds with stronger binding and higher specificity than random compounds, " + "and the two-compounds-per-gene criterion over-represents better-studied ones. A single " + "concentration was used throughout, which the authors call not ideal, so no dose-response " + "is established. It was generated in a single experiment at a single facility, which the " + "authors say limits the generalizability of any model trained on it. Two cell lines and " + "two time points support no claim of generality, and the scope is morphological profiling " + "only -- no mechanism of action is determined" + ), + scope=Scope( + summary="one imaging experiment at one facility, published as data for evaluating methods", + material="U2OS and A549 cell lines in 384-well plates", + duration_verbatim="not reported as a duration", + continuous_hours=None, + replicated=False, + sustained=False, + human_in_loop="the perturbation set was curated by hand from existing libraries", + ), + ), + Evidence( + key="ncats", + citation=( + "Iversen PW, Beck B, Chen Y-F, Dere W, Devanarayan V, Eastwood BJ, Farmen MW, Iturria SJ, " + "Montrose C, Moore RA, Weidner JR, Sittampalam GS. 'HTS Assay Validation.' In: Assay " + "Guidance Manual. Bethesda (MD): Eli Lilly & Company and the National Center for " + "Advancing Translational Sciences; 2012 May 1, last revised 2012 Oct 1. PMID 22553862; " + "Bookshelf ID NBK83783" + ), + url="https://www.ncbi.nlm.nih.gov/books/NBK83783/", + domain=Domain.CELL_ASSAY, + kind=EvidenceKind.GUIDANCE, + year=2012, + confidence=Confidence.CONFIRMED, + shorthand="the HTS assay-readiness guidance", + demonstrates=( + "Recommended practice for deciding an assay is ready to enter production screening. The " + "acceptance criterion is signal window >= 2 OR Z' >= 0.4 on all plates. Plate-uniformity " + "studies run over 3 days at Max/Mid/Min signal levels, 3 plates per day interleaved or 6 " + "per day in the concentration-response plus uniform-signal format. Replicate-experiment " + "studies run 20-30 compounds spanning the concentration range in each of two runs. Drift " + "and edge effects below 20 percent are considered insignificant; the worked example calls " + "26 percent drift a cause to investigate. The same acceptance block sets CVmax and CVmid " + "<= 20 percent, normalized SDmid <= 20, MSR < 3, limits of agreement between 0.33 and " + "3.0, and an outlier rate below 2 percent" + ), + does_not_establish=( + "The authors' own limit: the replicate-experiment study is a diagnostic and decision tool " + "used to establish that an assay is ready to go into production, and is not intended as a " + "substitute for post-production monitoring or to provide an estimate of the overall " + "minimum significant ratio. Beyond that, it establishes that an assay is technically fit " + "to screen and establishes nothing about whether a hit it finds is biologically real or " + "reproducible in another system. It is not a regulatory validation standard -- an " + "industry and NIH best-practice manual, not GLP, GMP, CLIA or FDA guidance, conferring no " + "compliance, and not covering clinical or diagnostic assay validation. The 0.4 threshold " + "is this manual's recommendation for this context and is not a universal cross-domain " + "quality bar; it should not be conflated with the more widely cited convention of Z' > " + "0.5 for an excellent assay. The manual leaves room for deviation rather than presenting " + "its numbers as immutable, directing that alternatives be discussed with a statistician. " + "Its worked designs are illustrated on 96-well plates and it does not validate these " + "criteria for every plate density or detection technology" + ), + scope=Scope( + summary="pre-production readiness criteria for biochemical and cell-based screening assays", + material="biochemical and cell-based screening assays, illustrated on 96-well plates", + duration_verbatim="plate-uniformity studies run over 3 days", + continuous_hours=None, + replicated=False, + sustained=False, + human_in_loop="the manual repeatedly directs borderline cases to a statistician", + ), + unverified=( + "no DOI exists for this chapter; Assay Guidance Manual chapters are not assigned one, " + "and any citation supplying a DOI for it should be treated as suspect", + "the editor list in effect at the 2012 revision, which is therefore omitted from the " + "citation rather than asserted", + ), + ), + Evidence( + key="smartkage", + citation=( + "Ho H, Kejzar N, Sasaguri H, Saito T, Saido TC, De Strooper B, Bauza M, Krupic J. 'A " + "fully automated home cage for long-term continuous phenotyping of mouse cognition and " + "behavior.' Cell Reports Methods, 2023;3(7):100532. doi:10.1016/j.crmeth.2023.100532" + ), + url="https://doi.org/10.1016/j.crmeth.2023.100532", + domain=Domain.IN_VIVO, + kind=EvidenceKind.EMPIRICAL_DEMONSTRATION, + year=2023, + confidence=Confidence.CONFIRMED, + shorthand="a single-mouse home-cage phenotyping study", + demonstrates=( + "Automated cognitive testing in the home cage -- T-maze alternation, novel object " + "recognition and object-in-place recognition -- combined with monitoring of locomotion, " + "drinking and quiescence patterns over long periods, with mice learning the tasks rapidly " + "and all testing carried out with no food or water restrictions. Lesion arm: hippocampal " + "n=5, medial entorhinal cortex n=4, control n=9 plus ten additional unlesioned C57BL/6J " + "mice, roughly 1 month in cage before lesioning and 2 months after. Classification " + "accuracy 100 percent (5/5) for hippocampal lesions and 25 percent (1/4) for medial " + "entorhinal lesions. Alzheimer arm: 5 AppNL-G-F knock-in and 3 age-matched controls " + "tested for about 4.5 months, genotype predicted at 80 percent (4/5) against 60 percent " + "(3/5) for standard T-maze, NOR and OPR" + ), + does_not_establish=( + "Author-stated limitations. The study is underpowered: the number of mice per group is " + "limited except for the control group, larger groups are conceded to be needed for the " + "entorhinal and Alzheimer arms, and all experiments came from a single animal facility " + "with no multi-site replication. The entorhinal result is a near-failure rather than a " + "success -- 25 percent (1/4) correctly classified, the rest misclassified as controls -- " + "so citing the system as discriminating hippocampal, entorhinal and sham lesions " + "overclaims: it separates hippocampal lesions essentially perfectly and entorhinal " + "lesions barely at all. It is not reproducible end to end, because the neural-network " + "tracking training data is proprietary and cannot be released, so users must build their " + "own. Recognition tasks were run only at inter-trial intervals up to a few minutes and " + "the authors state longer intervals must be explored to reach the impairments predicted " + "from standard tests, so no consolidated-memory claim is established. The cage tests one " + "isolated mouse at a time, which the authors themselves call suboptimal for stress and " + "low throughput, so it establishes nothing about group-housed or social phenotyping. The " + "Alzheimer result is genotype prediction from behavioral clustering in one knock-in model " + "and does not establish diagnostic utility, generalization to other models, or detection " + "of onset timing" + ), + scope=Scope( + summary="one isolated mouse per cage, one facility, small groups", + material="C57BL/6J and AppNL-G-F knock-in mice, singly housed", + duration_verbatim="about 4.5 months of continuous testing in the Alzheimer arm", + continuous_hours=None, + replicated=False, + sustained=False, + human_in_loop=( + "lesioning and surgery are bench procedures, and users must build their own tracking " + "training set because the authors' cannot be released" + ), + ), + unverified=( + "the composition of the 27/27 control cluster cannot be reconciled to a mouse count; " + "named control groups total 19 unlesioned mice, so '27 control mice' is not stated", + "the entorhinal breakdown is internally inconsistent -- 25 percent (1/4) implies three " + "misclassified but the text says two remaining mice were misclassified as controls -- so " + "only the headline 1/4 is stated", + "whether 'continuously tested for ~4.5 months' means uninterrupted; one retrieval " + "reported two testing periods separated by 8 months", + "the total number of animals across the study, which is not stated in any single " + "confirmable sentence, so no aggregate N is published here", + "the paper's own system name is 'smart-Kage', and full text at cell.com and bioRxiv both " + "returned HTTP 403, so verification rests on the PMC author manuscript", + ), + ), + Evidence( + key="homecage_multi", + citation=( + "Eswaraka, J., Gommet, C., Diomaiuta, D., Rigamonti, M., Rosati, G., Gaburro, S., Zwick, " + "M., Begoud, L., Warot, X. & Doenlen, R. 'Enhanced health evaluation in mice using " + "continuous home-cage monitoring and machine learning: a multicentric study.' Lab Animal " + "55, 275-281 (2026). doi:10.1038/s41684-026-01745-2" + ), + url="https://www.nature.com/articles/s41684-026-01745-2", + domain=Domain.IN_VIVO, + kind=EvidenceKind.EMPIRICAL_DEMONSTRATION, + year=2026, + confidence=Confidence.CONFIRMED, + shorthand="a retrospective analysis of 1,367 cages", + demonstrates=( + "Retrospective analysis of locomotion data from three institutions using the same " + "capacitance-based under-cage sensor platform, with machine-learning models generating " + "digital welfare alerts. A total of 1,367 cages across three sites over distinct study " + "periods -- approximately 300 days each at two sites and over 1,000 days at the third -- " + "yielding 90,540, 51,094 and 15,367 unique cage evaluations, 157,001 in total. The " + "algorithm identified animals in distress 3 to 6 days before verifiable clinical signs or " + "death were noticed, with accuracy of 66-80 percent at day -3 and 80-91 percent at day " + "-6, against the institutions' own clinical records as comparator" + ), + does_not_establish=( + "It does not establish that acting on the alerts improves welfare or outcomes. It is " + "retrospective -- locomotion anomalies preceded already-recorded clinical events -- with " + "no prospective or interventional validation that earlier intervention reduced suffering " + "or mortality; the benefits are framed as what the findings suggest, not demonstrate. " + "Author-stated limits: the algorithm missed 9-20 percent of verified clinical cases; more " + "robust datasets are needed to increase accuracy; alerts are not delivered in real time, " + "as welfare alerts are not reported to caretakers until the morning; the sample size of " + "clinical cases per category is too small to draw conclusions on specific patterns; " + "cross-site comparison is unreliable because sites recorded clinical cases in different " + "categories; the technology cannot track individual animal locomotor activity in " + "group-housed cages and therefore likely lacks the sensitivity to detect every " + "spontaneous death; and it measures only locomotor activity, while not all clinical " + "conditions affect locomotion. Scope is mice only, one proprietary vendor platform, and " + "cage-level rather than animal-level resolution. The accuracy figures are inseparable " + "from a tolerated false-positive rate -- the headline range corresponds to a model tuned " + "to a 15 percent maximum false-positive threshold -- so quoting accuracy without the " + "false-positive budget overclaims. Four co-authors are affiliated with the company that " + "develops and sells the platform being evaluated" + ), + scope=Scope( + summary="retrospective, cage-level, one vendor platform, no interventional arm", + material="group-housed mice in ventilated cages", + duration_verbatim="approximately 300 days at two sites and over 1,000 days at the third", + continuous_hours=None, + replicated=True, + sustained=False, + human_in_loop=( + "the comparator is clinical records kept by staff, and alerts are not delivered to " + "caretakers until the morning" + ), + ), + unverified=( + "per-institution clinical case counts of 229, 42 and 65, extracted from full text but " + "not re-confirmable as a verbatim quote", + "per-category detection rates including found-dead at 93/85/100 percent, hunched posture, " + "eye issues and weight loss at 100 percent, injuries 73 percent, fighting 62 percent and " + "skin issues 57 percent; the paper says these subgroups are too small to support " + "conclusions, so none are stated", + "day-0 accuracy of roughly 17.9-50.8 percent and the 5/15/30 percent false-positive " + "thresholds separating the three models", + "the exact number of individual mice, which the paper does not appear to state", + "an 'over 50 percent' operational-efficiency figure in press coverage that could not be " + "traced to any result in the paper", + ), + ), + Evidence( + key="arrive", + citation=( + "Percie du Sert, N. et al. 'The ARRIVE guidelines 2.0: Updated guidelines for reporting " + "animal research.' PLOS Biology 18(7): e3000410 (2020). doi:10.1371/journal.pbio.3000410" + ), + url="https://doi.org/10.1371/journal.pbio.3000410", + domain=Domain.IN_VIVO, + kind=EvidenceKind.GUIDANCE, + year=2020, + confidence=Confidence.CONFIRMED, + shorthand="the ARRIVE 2.0 reporting checklist", + demonstrates=( + "A reporting framework for in vivo experiments, structured as a 21-item checklist split " + "by a Delphi prioritization exercise into the ARRIVE Essential 10 -- study design (1), " + "sample size (2), inclusion and exclusion criteria (3), randomization (4), " + "blinding/masking (5) and outcome measures (6) among them -- and an 11-item Recommended " + "Set describing research context, which is where housing and husbandry (15) and animal " + "care and monitoring (16) sit. Item 14 asks authors to name the ethical review committee " + "that approved the study and any licence or protocol numbers, or to justify the absence " + "of approval" + ), + does_not_establish=( + "It governs what is reported in a manuscript, not how research is designed, conducted, " + "approved, or whether it is acceptable. It is not ethical or welfare authorization of any " + "kind: not an ethics review, not an institutional or regulatory licence, not a 3Rs " + "assessment, and adherence confers no approval to perform any procedure -- item 14 only " + "asks authors to name the committee that already approved the work. It does not establish " + "that a study is methodologically sound: it is a reporting guideline, not a risk-of-bias " + "or quality-assessment instrument, so full compliance means a study is transparently " + "described, not that it is unbiased, adequately powered or valid; formal appraisal " + "requires separate tools. It prescribes no experimental design, no husbandry standard, no " + "humane endpoint and no monitoring frequency. It does not establish its own " + "effectiveness: it is a consensus-derived checklist with no evidence presented that " + "adopting it improves reporting quality, reproducibility or welfare outcomes. It carries " + "no enforcement mechanism -- adoption is voluntary and mediated by journals. And because " + "husbandry and monitoring sit in the Recommended Set, the ARRIVE minimum does not itself " + "compel reporting them" + ), + scope=Scope( + summary="what a manuscript must disclose about an animal experiment already performed", + material="reported in vivo experiments, any species", + duration_verbatim="not applicable", + continuous_hours=None, + replicated=False, + sustained=False, + human_in_loop="the entire instrument is a checklist completed by authors and journals", + ), + unverified=( + "that the authors state in so many words that ARRIVE is not welfare approval or not a " + "risk-of-bias tool; both readings follow directly from item 14 and the reporting-only " + "framing, but no verbatim disclaimer sentence was located, so this is a characterization " + "of the guideline's function and not an author-stated limitation", + "Delphi process specifics -- round count and participant numbers or nationalities -- " + "which are therefore not stated", + ), + ), + Evidence( + key="alabos", + citation=( + "Fei, Y.; Rendy, B.; Kumar, R.; Dartsi, O.; Sahasrabuddhe, H. P.; McDermott, M. J.; Wang, " + "Z.; Szymanski, N. J.; Walters, L. N.; Milsted, D.; Zeng, Y.; Jain, A.; Ceder, G. " + "'AlabOS: a Python-based reconfigurable workflow management framework for autonomous " + "laboratories.' Digital Discovery, 2024, 3(11), 2275-2288. doi:10.1039/D4DD00129J" + ), + url="https://doi.org/10.1039/D4DD00129J", + domain=Domain.ORCHESTRATION, + kind=EvidenceKind.RESOURCE, + year=2024, + confidence=Confidence.CONFIRMED, + shorthand="one prototype lab's workflow manager, ~3,500 samples over about 1.5 years", + demonstrates=( + "Orchestration middleware for an autonomous laboratory: a graph-based workflow model with " + "tasks as nodes and dependencies as edges; a central resource manager that tracks device " + "status and allocates resources per task so concurrent tasks cannot conflict, with task " + "actors required to request occupancy of devices and sample positions before acting; " + "sample-position tracking where a position holds one sample at a time, positions update " + "as robots or humans move material, position is set to null on leaving the lab and full " + "history is preserved; and a user-request module that asks for explicit human " + "intervention through dashboard, chat bot or email, cited for robot-arm error recovery " + "and consumable replacement. Around 3,500 samples were processed over about 1.5 years, " + "with a peak daily throughput of 149 samples. Resource allocation is first-come-first-" + "serve with task priority considered and no estimate of task duration" + ), + does_not_establish=( + "The ~3,500 figure is a deployment usage statistic, not a scientific-success metric: it " + "counts samples the software drove through synthesis and characterization, and does not " + "mean 3,500 successful syntheses, novel materials or validated discoveries. Validation is " + "single-lab and single-domain -- one prototype laboratory doing inorganic solid-state " + "powder synthesis, with no multi-site deployment, no cross-domain evidence and no " + "demonstration that the framework generalizes beyond the lab that built it. There is no " + "controlled comparison or benchmark against other workflow managers on throughput, " + "reliability or developer effort, so no claim of superiority is supported. It does no " + "experiment design, active learning or discovery decision-making; the authors explicitly " + "delegate that to external planning software. It is the layer beneath the science. " + "Scheduling is deliberately simple and self-declared as non-optimal, with the authors " + "stating that more sophisticated reservation techniques could be integrated to further " + "optimize throughput -- that is future work, not a result. And autonomy here is not " + "unattended operation: the human-intervention machinery is load-bearing, error recovery " + "depends on pre-implemented routines, and the authors concede that failures from worn " + "hardware or newly emerged software exceptions not caught in the soak test still require " + "operators" + ), + scope=Scope( + summary="one prototype laboratory, one synthesis domain, human-in-the-loop error recovery", + material="inorganic solid-state powder synthesis samples", + duration_verbatim="around 3,500 samples synthesized over 1.5 years", + continuous_hours=None, + replicated=False, + sustained=False, + human_in_loop=( + "a dedicated module requests human intervention, and unanticipated hardware or software " + "failures still require operators" + ), + ), + ), + Evidence( + key="pylabrobot", + citation=( + "Wierenga, R. P., Golas, S. M., Ho, W., Coley, C. W., and Esvelt, K. M. (2023). " + "'PyLabRobot: An open-source, hardware-agnostic interface for liquid-handling robots and " + "accessories.' Device 1(4), 100111. doi:10.1016/j.device.2023.100111" + ), + url="https://doi.org/10.1016/j.device.2023.100111", + domain=Domain.ABSTRACTION, + kind=EvidenceKind.RESOURCE, + year=2023, + confidence=Confidence.PARTIAL, + shorthand="a hardware-agnostic control interface over three robot families", + demonstrates=( + "An open-source, cross-platform Python interface capable of programming diverse " + "liquid-handling robots: a single front-end layer over hardware-specific execution-layer " + "backends, with a universal set of commands and representations for deck layout and " + "labware. Hardware agnosticity is demonstrated rather than asserted -- labware " + "definitions from three vendors were translated into the hardware-agnostic model, and " + "liquid handler resources were used on an Opentrons robot and vice versa. Three robot families " + "at publication -- liquid handler/STARLet, Tecan Freedom EVO, Opentrons OT-2 -- plus BMG " + "plate readers as accessories. Operating-system coverage is asymmetric: the STAR backend " + "was verified on Windows, macOS and Raspberry Pi OS, the EVO backend only on Windows and " + "macOS. A browser-based simulator ships with the package" + ), + does_not_establish=( + "No quantitative liquid-handling performance validation. The paper presents zero accuracy " + "or precision data, no gravimetric validation, no coefficients of variation and no " + "throughput benchmarks, and never compares PyLabRobot-driven pipetting against vendor " + "software -- so it does not establish that protocols run through it match vendor-native " + "accuracy. Testing stops at command generation rather than physical outcome: unit tests " + "ensure that user input produces consistent machine-level commands and, where possible, " + "that they are transmitted correctly, which is correctness at the firmware-string layer, " + "not at the liquid-in-the-well layer. The simulator does not replace hardware, by the " + "authors' own statement that some facets of development will require physical access to a " + "robot. Backend parity is not established -- STAR and EVO use direct firmware command " + "strings over USB while Opentrons goes through an HTTP API and must mirror a second deck " + "model, so hardware-agnostic does not mean equivalent capability on every backend. " + "Coverage is three robot families plus plate readers, not all liquid handlers. There is " + "no claim of scheduling, protocol optimization, error recovery or closed-loop autonomy, " + "and no user study, adoption metric or cross-lab reproducibility study" + ), + scope=Scope( + summary="a control abstraction, validated at the command layer and not at the bench", + material="none; the artifact is software", + duration_verbatim="not applicable", + continuous_hours=None, + replicated=False, + sustained=False, + human_in_loop="a developer writes the protocol and, for a new robot, the backend", + ), + unverified=( + "that simulation was demonstrated across multiple liquid-handler families. The paper " + "makes an architectural assertion -- the simulator works with any robot for which a deck " + "model exists -- and immediately qualifies it: a developer must still enforce " + "machine-specific constraints such as pipetting-channel configuration. No run across all " + "three families is reported, so the safe wording is a simulator designed to work with any " + "robot for which a deck model exists", + "any unit-test count or code-coverage percentage; the paper reports none, so any such " + "figure attributed to it would be fabricated", + "any pipetting accuracy, precision, throughput or timing number; the paper reports none", + "verbatim body text was checked against the bioRxiv preprint on PMC because the version " + "of record is paywalled, so preprint-to-record wording changes could not be ruled out", + ), + ), + Evidence( + key="iso23783", + citation=( + "ISO 23783-1:2022, Automated liquid handling systems -- Part 1: Vocabulary and general " + "requirements (first edition, 2022-08; ISO/TC 48; 19 pp.); and ISO 23783-3:2022, " + "Automated liquid handling systems -- Part 3: Determination, specification and reporting " + "of volumetric performance (2022-08; ISO/TC 48; 21 pp.)" + ), + url="https://www.iso.org/standard/76952.html", + domain=Domain.ABSTRACTION, + kind=EvidenceKind.STANDARD, + year=2022, + confidence=Confidence.CONFIRMED, + shorthand="the automated-liquid-handling volumetric standard", + demonstrates=( + "Part 1 defines terms relating to automated liquid handling systems and specifies general " + "requirements for their use, describing system types, use requirements, settings, " + "adjustments and environmental requirements. Part 3 covers collecting and examining " + "volumetric performance data, indexing and tracking it, descriptive statistics for " + "evaluating it, and reporting requirements. Both are the work of ISO/TC 48, and the " + "series cancels and replaces IWA 15:2015" + ), + does_not_establish=( + "The most load-bearing limit is what these two parts leave out. Part 2 holds the " + "measurement procedures for determining volumetric performance, and Part 1 says so " + "explicitly in a note to its own scope. Citing Parts 1 and 3 to support any claim about " + "HOW volumetric performance is measured does not work -- Part 3 governs only what is done " + "with data after it is collected. Applicability is narrow and stated in both scopes: " + "complete, installed liquid handling devices including tips and other essential parts, " + "performing liquid handling tasks without human intervention into labware. It therefore " + "does not cover manual pipettes, incomplete or uninstalled devices, or operations " + "requiring human intervention. Neither part establishes acceptance criteria, pass/fail " + "tolerance limits or accuracy grades for any application or assay. The series certifies " + "no instrument, vendor or protocol. ISO standards are voluntary, not regulatory " + "requirements. Part 1 adds that the tests it establishes should be carried out by trained " + "personnel. And the series addresses volumetric performance only -- trueness and " + "precision of delivered volume -- not biological assay validity, contamination or " + "carryover, sterility, throughput, or the scientific reproducibility of results" + ), + scope=Scope( + summary="volumetric trueness and precision of installed automated liquid handlers", + material="none; the artifact is a specification", + duration_verbatim="not applicable", + continuous_hours=None, + replicated=False, + sustained=False, + human_in_loop="the standard directs that its tests be carried out by trained personnel", + ), + unverified=( + "the normative content of Part 3, which is paywalled and had no retrievable free " + "preview; verification rests on its official title as quoted inside the published Part 1, " + "its catalogue metadata and its published abstract, so no specific requirement, formula " + "or tolerance from Part 3 is stated here", + "the edition statement for Part 3, inferred from the series structure rather than read " + "off its cover page", + "the substantive text of Part 1 clauses 6 and 7 on testing, calibration and specification " + "of volumetric performance, which the free preview does not reach", + ), + ), + Evidence( + key="provo", + citation=( + "Lebo, T., Sahoo, S., McGuinness, D. (eds.). 'PROV-O: The PROV Ontology.' W3C " + "Recommendation, 30 April 2013. World Wide Web Consortium. No DOI; the dated W3C URI is " + "the canonical identifier" + ), + url="https://www.w3.org/TR/2013/REC-prov-o-20130430/", + domain=Domain.PROVENANCE, + kind=EvidenceKind.STANDARD, + year=2013, + confidence=Confidence.CONFIRMED, + shorthand="the PROV-O provenance ontology", + demonstrates=( + "An OWL2 ontology providing classes, properties and restrictions to represent and " + "interchange provenance information generated in different systems and under different " + "contexts. The three starting-point classes are prov:Entity, prov:Activity and prov:Agent; " + "generation, use, derivation and association each appear twice over, as an object property " + "and as a qualified class -- prov:wasGeneratedBy / prov:Generation, prov:used / " + "prov:Usage, prov:wasDerivedFrom / prov:Derivation, prov:wasAssociatedWith / " + "prov:Association. The normative OWL file declares 30 classes, 44 object properties, 6 " + "datatype properties and 17 annotation properties; the class count matches the document's " + "own split of 3 starting-point, 7 expanded and 20 qualified terms" + ), + does_not_establish=( + "PROV-O does not establish validity and enforces no constraints. The uniqueness, " + "event-ordering, impossibility and type constraints that define a valid PROV instance " + "live in the separate PROV-CONSTRAINTS Recommendation; PROV-O supports OWL2 entailments " + "and does not check validity, so citing it as authority for provenance validation or " + "consistency checking is an overclaim. It is also a category error to call it the " + "provenance model: PROV-O, PROV-XML and PROV-N are serializations of a conceptual model " + "that lives in PROV-DM. Exactly five of its axioms fall outside the OWL 2 RL profile, and " + "the spec notes it cannot assume everybody is using OWL reasoning, so it does not " + "guarantee reasoner-derived completeness. Bundles get no RDF encoding here. The inverse of " + "prov:wasDerivedFrom is not defined; only recommended names are supplied, " + "non-normatively. Location modeling is out of scope. Responsibility is not apportioned -- " + "agents are responsible in some way without the ontology saying who bears it or to what " + "degree. And provenance is not trust, quality or accuracy: a PROV record is an input to " + "such an assessment, not a determination of it, validity means logical consistency rather " + "than factual accuracy, and a fully valid instance can record a false history" + ), + scope=Scope( + summary="an RDF/OWL vocabulary for exchanging provenance, with validity left elsewhere", + material="none; the artifact is a vocabulary", + duration_verbatim="not applicable", + continuous_hours=None, + replicated=False, + sustained=False, + human_in_loop="nothing collects or generates the provenance; that is out of scope", + ), + unverified=( + "the exact starting-point property count, variously presented as 9, 10 or 11 depending on " + "whether prov:type and prov:label are folded in, and the expanded/qualified property " + "split, which does not cleanly reconcile with the parsed totals; no figure is stated", + ), + ), + Evidence( + key="fda_di", + citation=( + "U.S. Food and Drug Administration, 'Data Integrity and Compliance With Drug CGMP: " + "Questions and Answers -- Guidance for Industry,' December 2018 (Final, Level 1). Docket " + "No. FDA-2018-D-3984. Issued by CDER in cooperation with CBER, CVM and ORA. No DOI" + ), + url=( + "https://www.fda.gov/regulatory-information/search-fda-guidance-documents/" + "data-integrity-and-compliance-drug-cgmp-questions-and-answers" + ), + domain=Domain.PROVENANCE, + kind=EvidenceKind.GUIDANCE, + year=2018, + confidence=Confidence.CONFIRMED, + shorthand="the CGMP data-integrity questions and answers", + demonstrates=( + "Eighteen numbered questions across 17 pages setting out the agency's expectations for " + "data integrity under 21 CFR parts 210, 211 and 212. Data are expected to be reliable and " + "accurate, and data integrity is defined as attributable, legible, contemporaneously " + "recorded, original or a true copy, and accurate. Metadata is the contextual information " + "required to understand data and must be retained so a CGMP activity can be reconstructed. " + "An audit trail is a secure, computer-generated, time-stamped electronic record allowing " + "reconstruction of the creation, modification or deletion of an electronic record. The " + "system administrator role, including rights to alter files and settings, should be " + "assigned to personnel independent from those responsible for record content. Processes " + "should be designed so that data cannot be modified without a record of the modification. " + "A backup is a true copy maintained securely through the retention period and must be " + "exact, complete and secure from alteration -- temporary copies do not satisfy this. " + "Audit-trail review falls to the personnel responsible for CGMP record review" + ), + does_not_establish=( + "The guidance is explicitly nonbinding and establishes no requirement of its own. Its own " + "words: it represents the agency's current thinking, does not establish any rights for " + "any person, is not binding on FDA or the public, and an alternative approach may be used " + "if it satisfies the applicable statutes and regulations. Every page is headed 'Contains " + "Nonbinding Recommendations'. The binding force lives in 21 CFR 210, 211 and 212, so any " + "citation implying this document requires or mandates something overclaims. Product scope " + "is drugs including biologics, specifically finished pharmaceuticals and PET drugs: it " + "does not cover medical devices or the quality system regulation, does not cover good " + "clinical practice trial data or nonclinical laboratory data, and does not directly cover " + "active pharmaceutical ingredients. It does not establish or reinterpret 21 CFR part 11 " + "electronic-records requirements, and expressly notes part 11 remains subject to the " + "agency's reexamination. It deliberately declines to set fixed technical standards -- on " + "audit-trail review frequency it prescribes no interval and defers to the firm's own " + "knowledge of its processes and risk assessment -- so it establishes no mandated review " + "frequency, retention technology or access-control implementation" + ), + scope=Scope( + summary="nonbinding expectations for electronic records under drug CGMP", + material="none; the artifact is agency guidance", + duration_verbatim="not applicable", + continuous_hours=None, + replicated=False, + sustained=False, + human_in_loop="the quality unit reviews and approves production and control records", + ), + ), + Evidence( + key="labsafety", + citation=( + "Zhou, Y., Yang, J., Huang, Y., Guo, K., Emory, Z., Ghosh, B., Bedar, A., Shekar, S., " + "Liang, Z., Chen, P.-Y., Gao, T., Geyer, W., Moniz, N., Chawla, N. V., & Zhang, X. " + "(2026). 'Benchmarking large language models on safety risks in scientific " + "laboratories.' Nature Machine Intelligence, 8, 20-31. doi:10.1038/s42256-025-01152-1" + ), + url="https://doi.org/10.1038/s42256-025-01152-1", + domain=Domain.SAFETY, + kind=EvidenceKind.BENCHMARK, + year=2026, + confidence=Confidence.CONFIRMED, + shorthand="a text benchmark of 19 models on constructed lab-safety questions", + demonstrates=( + "Nineteen models evaluated -- 8 proprietary and 7 open-weight language models plus 4 " + "open-weight vision-language models -- against 765 multiple-choice questions (632 " + "text-only, 133 with an image), 404 realistic laboratory scenarios and 3,128 open-ended " + "tasks. The authors' headline finding is that no evaluated model surpasses 70 percent " + "accuracy on the Hazards Identification Test. The contrast that makes it a finding: on " + "the multiple-choice portion GPT-4o reached 86.55 percent text-only with zero-shot " + "chain-of-thought and DeepSeek-R1 reached 84.49 percent. The structural result is the gap " + "-- proprietary models do well on structured questions and show no clear advantage in " + "open-ended reasoning" + ), + does_not_establish=( + "The 70 percent figure is sub-task specific and is the biggest trap in this citation. It " + "refers to the open-ended, scenario-based Hazards Identification Test only and is not " + "overall model performance; writing 'no model exceeded 70 percent accuracy' without " + "naming the sub-task invites the reader to conclude models score under 70 percent on lab " + "safety generally, which the paper does not show. The benchmark measures text-based " + "knowledge and reasoning on constructed questions and scenarios. It establishes nothing " + "about real-world laboratory behavior, nothing about physical task execution, nothing " + "about accident or injury rates, and it does not show that model use has caused any lab " + "incident. Domain scope is biology, chemistry and physics at Biosafety Levels 1 to 3, with " + "BSL-4 explicitly excluded, and the questions are aligned to US OSHA protocols, so results " + "do not transfer automatically to other regulatory regimes. The human baseline is weak by " + "the authors' own admission -- some participants were junior researchers who may not " + "represent experienced experts -- so any 'outperforms humans' framing carried over from " + "the preprint should be dropped. Model coverage is a late-2024 to early-2025 snapshot and " + "says nothing about current frontier models. Note also that 'LabSafety Bench' is the " + "benchmark's name and the preprint title, not the title of the published paper; pairing " + "the preprint title with the Nature DOI is a mismatch, and the preprint's variant figures " + "-- 520 scenarios, 4,090 open-ended questions, 'none above 75 percent' -- still circulate " + "and contradict the published ones" + ), + scope=Scope( + summary="constructed questions and scenarios scored offline; no laboratory was involved", + material="none; the subjects are models, not material", + duration_verbatim="not applicable", + continuous_hours=None, + replicated=False, + sustained=False, + human_in_loop="a human baseline was collected, and the authors call it weak", + ), + unverified=( + "the per-model hazard-identification score table and the paper's full limitations " + "section could not be inspected directly; the 70 percent figure is verified as the " + "authors' abstract-level assertion and corroborated by press coverage, but the underlying " + "per-model numbers were not independently checked", + "the grading methodology for the 3,128 open-ended tasks -- human expert or model-as-judge " + "-- which could not be confirmed from any source reached. This matters, because the 70 " + "percent headline rests on however those free-text responses were scored", + "a reported 5-10 percent gain from fine-tuning, which appears only in secondary press " + "coverage and is not confirmed against the paper", + ), + ), +) + + +EVIDENCE_BY_KEY: Dict[str, Evidence] = {e.key: e for e in EVIDENCE} + + +def by_domain(domain: Domain) -> Tuple[Evidence, ...]: + return tuple(e for e in EVIDENCE if e.domain is domain) + + +def partially_verified() -> Tuple[Evidence, ...]: + """Entries where a number this module would otherwise state could not be confirmed.""" + return tuple(e for e in EVIDENCE if not e.verified) + + +# -- claim against literature --------------------------------------------------- + + +@dataclass(frozen=True) +class ScopeGap: + """One cited work measured against one claim. + + `shortfalls` is a list rather than a verdict because the ways a citation falls short are + not interchangeable. A work in the wrong domain needs a different citation. A work in the + right domain that ran for fifteen hours needs a longer run. Collapsing both to "not + supported" throws away the only part of the answer that tells anyone what to do next. + """ + + evidence: Evidence + covers: bool + shortfalls: Tuple[str, ...] + reason: str + + +@dataclass(frozen=True) +class Support: + """What the published field actually says about one claim. + + `speaks_to` is restricted to the claim's own domain. That is the harsh cut and it is the + one that does the work: the citations that get misused most are not weak, they are strong + evidence about a different kind of matter. + """ + + claim_domain: Domain + scope: Scope + speaks_to: Tuple[ScopeGap, ...] + out_of_domain: Tuple[ScopeGap, ...] + + @property + def supported(self) -> bool: + """True only if some in-domain work demonstrated everything the claim asserts. + + An empty `speaks_to` is False, not vacuously True. A claim no cited work addresses is + unsupported in the strongest sense available, and the natural implementation -- all() over + an empty sequence -- returns True, which would report the least evidenced claims in the + table as the best supported ones. + """ + return any(g.covers for g in self.speaks_to) + + def gaps(self) -> Tuple[ScopeGap, ...]: + return tuple(g for g in self.speaks_to if not g.covers) + + def reason(self) -> str: + if self.supported: + covering = ", ".join(g.evidence.key for g in self.speaks_to if g.covers) + return f"demonstrated within scope by {covering}" + field = self.claim_domain.value.replace("_", " ") + if not self.speaks_to: + return ( + f"no cited work operates in {field} at all; the claim rests on citations from " + f"{', '.join(sorted({g.evidence.domain.value.replace('_', ' ') for g in self.out_of_domain}))}" + ) + return "; ".join(g.reason for g in self.gaps()) + + +# How each non-demonstration describes itself, for the one sentence that refuses it. A bare +# enum value reads as a category label rather than an answer, and this report is read by +# people deciding whether to keep citing something. +_KIND_PHRASE: Dict[EvidenceKind, str] = { + EvidenceKind.BENCHMARK: "a benchmark for evaluating models or methods", + EvidenceKind.GUIDANCE: "recommended practice", + EvidenceKind.STANDARD: "a normative specification", + EvidenceKind.RESOURCE: "a software or data artifact", +} + + +def _shortfalls(evidence: Evidence, scope: Scope) -> Tuple[str, ...]: + """Every specific way one work falls short of one claimed scope. + + Kind is checked first and returns alone, because the remaining comparisons are only + meaningful against something that ran. Measuring a duration gap, a replication gap and a + human-in-the-loop gap against a reporting checklist would produce four grammatical + sentences and one category error, and burying the real objection under three fabricated + ones is how a report earns the right to be skimmed. + """ + out: List[str] = [] + short = evidence.shorthand + demo = evidence.scope + + if not evidence.kind.demonstrates_operation: + return ( + f"{short} is {_KIND_PHRASE[evidence.kind]}, not a demonstration; it does not show a lab " + "operating, so it cannot evidence an operating claim", + ) + + if scope.sustained and not demo.sustained: + out.append( + f"{short} is a single bounded campaign and does not reach sustained production" + ) + + if scope.continuous_hours is not None: + if demo.continuous_hours is None: + out.append( + f"the claim asks for {scope.continuous_hours:g} h of continuous operation and {short} " + f"states no duration that converts to a figure ({demo.duration_verbatim})" + ) + elif demo.continuous_hours < scope.continuous_hours: + out.append( + f"{short} ran {demo.continuous_hours:g} h against the {scope.continuous_hours:g} h " + "the claim asserts" + ) + + if scope.replicated and not demo.replicated: + out.append( + f"{short} was not replicated, so its numbers describe that run rather than a capability" + ) + + if scope.unattended and demo.human_in_loop: + out.append(f"{short} kept a human in the loop: {demo.human_in_loop}") + + return tuple(out) + + +def support_for(claim_domain: Domain, scope: Scope) -> Support: + """What the literature supports for a claim in one domain at one operating scope. + + Returns both halves deliberately. `speaks_to` is what an honest citation list would contain. + `out_of_domain` is everything else, each carrying the reason it does not apply -- because + the works that get miscited are already known to whoever is making the claim, and a report + that silently omitted them would read as an oversight rather than as a refusal. + """ + speaks: List[ScopeGap] = [] + outside: List[ScopeGap] = [] + + for e in EVIDENCE: + if e.domain is claim_domain: + shortfalls = _shortfalls(e, scope) + speaks.append( + ScopeGap( + evidence=e, + covers=not shortfalls, + shortfalls=shortfalls, + reason="; ".join(shortfalls) if shortfalls else f"{e.shorthand} covers the claimed scope", + ) + ) + else: + field = claim_domain.value.replace("_", " ") + crossing = ( + " -- living material contaminates, loses viability and changes state while it waits, " + "and none of that has an analogue in the cited domain" + if claim_domain.living and not e.domain.living + else "" + ) + outside.append( + ScopeGap( + evidence=e, + covers=False, + shortfalls=(), + reason=( + f"{e.shorthand} is {e.domain.value.replace('_', ' ')} work; it does not speak to a " + f"{field} claim{crossing}" + ), + ) + ) + + return Support( + claim_domain=claim_domain, + scope=scope, + speaks_to=tuple(speaks), + out_of_domain=tuple(outside), + ) + + +# -- what nothing here establishes ---------------------------------------------- + + +@dataclass(frozen=True) +class UnsupportedClaim: + """A claim made routinely that no work in this table establishes. + + `nearest` names the citations usually offered for it. That field is what makes the entry + useful rather than merely negative: the argument is never "you have no evidence", it is + "here is the evidence you meant, and here is where it stops". `would_establish_it` keeps the + entry falsifiable and retirable, in the same spirit as `intelligence.Benchmark.how_to_measure` + -- an objection with no stated remedy is a position, not a finding. + """ + + name: str + claim: str + why_unsupported: str + nearest: Tuple[str, ...] + would_establish_it: str + + +UNSUPPORTED: Tuple[UnsupportedClaim, ...] = ( + UnsupportedClaim( + name="integrated_autonomy_at_production_rates", + claim=( + "an integrated autonomous laboratory runs a real workload end to end, without a human, " + "at a production rate" + ), + why_unsupported=( + "Every demonstration in this table stops short, and each stops in a different place. " + "A-Lab is one 17-day campaign on one instrument, on inorganic powders, unreplicated, and " + "the framework paper from the same group concedes that worn hardware and newly emerged " + "software exceptions still require operators. Dai et al. ran almost four consecutive days " + "and the one anomaly that mattered -- a cyclized isomer with the expected molecular mass " + "-- was caught by human inspection of the NMR and confirmed by off-workflow " + "single-crystal X-ray diffraction. RoboCulture ran 15 hours on a single plate with " + "seeding done by hand and the subculturing split manually initiated. Coley et al. require " + "an expert chemist to author a recipe file for every route. Nothing here demonstrates the " + "integration, the duration and the absence of a human at the same time" + ), + nearest=("a_lab", "dai", "roboculture", "coley", "alabos"), + would_establish_it=( + "a replicated multi-week campaign in the claimed domain, on the claimed material, with " + "the human interventions counted rather than described, and a denominator of attempted " + "runs so the success rate is a rate" + ), + ), + UnsupportedClaim( + name="nominal_speed_as_biological_throughput", + claim=( + "an instrument's rated or nominal speed is the rate at which a lab produces biological " + "results" + ), + why_unsupported=( + "No cited work reports a throughput advantage over manual work, and two decline the " + "framing outright. RoboCulture's authors position the system as operating more like an " + "autonomous vehicle tailored to individual or small-group use than for high-throughput " + "screening, and say it neither beats commercial liquid handlers on throughput nor claims " + "to. Dai et al. claim only that the algorithmic decisions are effectively instantaneous, " + "which is the decision step and not synthetic throughput. AlabOS's roughly 3,500 samples " + "over about 1.5 years is a deployment usage statistic, not a count of successes, and its " + "scheduler is first-come-first-serve with no task-duration modeling, which the authors " + "call non-optimal and future work. Rated speed is a property of a motion; a biological " + "result requires the incubation, the gate, and the rerun after the gate fails" + ), + nearest=("roboculture", "dai", "alabos"), + would_establish_it=( + "measured wall-clock time per completed, QC-passing result over a run long enough to " + "include failures and reruns, compared against the manual baseline it claims to beat" + ), + ), + UnsupportedClaim( + name="simulation_as_qualification", + claim="a protocol that runs correctly in simulation is qualified to run on real material", + why_unsupported=( + "PyLabRobot's own authors draw this line: unit tests ensure that user input produces " + "consistent machine-level commands and that they are transmitted correctly, which is " + "correctness at the firmware-string layer and not at the liquid-in-the-well layer, and " + "they state plainly that some facets of development will require physical access to a " + "robot. The paper reports no accuracy, precision or gravimetric data at all. The ISO " + "series that does address volumetric performance applies to complete, installed devices " + "and puts the measurement procedures in a part not cited here, while setting no " + "acceptance criteria for any assay. The HTS guidance puts assay readiness behind " + "plate-uniformity studies over 3 days and replicate-experiment studies on the instrument. " + "A simulator establishes that the commands are the intended commands" + ), + nearest=("pylabrobot", "iso23783", "ncats"), + would_establish_it=( + "a gravimetric or dye-based volumetric measurement at the working volume with the tips " + "actually used, followed by the plate-uniformity and replicate-experiment studies the " + "assay guidance specifies" + ), + ), + UnsupportedClaim( + name="ai_fluency_as_hazard_competence", + claim=( + "a model that answers laboratory questions well is competent to judge laboratory hazards" + ), + why_unsupported=( + "The one benchmark here that asks the question found the opposite, and found it as a gap " + "rather than a level: no evaluated model surpassed 70 percent accuracy on the Hazards " + "Identification Test while GPT-4o reached 86.55 percent on the multiple-choice portion. " + "Structured recall and open-ended hazard reasoning came apart. And even the low number is " + "the wrong kind of evidence for a bench claim: the benchmark measures text-based " + "knowledge on constructed questions, establishes nothing about real-world laboratory " + "behavior or physical task execution, covers Biosafety Levels 1 to 3 with BSL-4 excluded, " + "and is aligned to one regulatory regime" + ), + nearest=("labsafety",), + would_establish_it=( + "measured hazard-identification performance on this lab's own materials and procedures, " + "scored by someone qualified to sign off on them, with the miss rate reported rather than " + "the accuracy" + ), + ), + UnsupportedClaim( + name="capital_as_capacity", + claim="purchased instruments constitute laboratory capacity", + why_unsupported=( + "Nothing cited measures capacity as a function of spend, and in every demonstration the " + "binding constraint was somewhere else. A-Lab's was analysis quality -- the senior author " + "concedes a human can perform a higher-quality refinement -- and the validity of a blind " + "target. RoboCulture's was a fiducial-tag calibration its authors call fragile and " + "time-consuming to maintain, over a perception stack that does not generalize beyond well " + "plates. AlabOS's was error-recovery routines that have to be implemented in advance. " + "This repository makes the same point from the other side: six reverse-engineered " + "instruments are physically present on the bench and 0 of 54 seeded commands are decoded, " + "so the ledger reaches step 1 of 18 on an unattended run. The instruments were bought. " + "The capacity was not" + ), + nearest=("a_lab", "roboculture", "alabos"), + would_establish_it=( + "a completed protocol on the purchased instruments with the decode, calibration, " + "benchmarking and recovery work counted as part of the cost of the capacity" + ), + ), +) + + +def unsupported_claims() -> Tuple[UnsupportedClaim, ...]: + """The standing list of things no work in this table establishes. + + Standing, not computed. These are not gaps that a scope comparison discovers; they are the + five inferences this module was built after watching get made, and they stay in the table + until a citation retires one. `support_for` answers a question somebody thought to ask, and + this answers the ones nobody does. + """ + return UNSUPPORTED + + +def summary() -> Dict[str, int]: + """How much of the cited field is demonstration, and how much of it was fully checked.""" + return { + "evidence": len(EVIDENCE), + "demonstrations": sum(1 for e in EVIDENCE if e.kind.demonstrates_operation), + "confirmed": sum(1 for e in EVIDENCE if e.verified), + "partial": sum(1 for e in EVIDENCE if not e.verified), + "entries_with_unverified_numbers": sum(1 for e in EVIDENCE if e.unverified), + "domains_covered": len({e.domain for e in EVIDENCE}), + "unsupported_claims": len(UNSUPPORTED), + } + + +__all__ = [ + "EVIDENCE", + "EVIDENCE_BY_KEY", + "UNSUPPORTED", + "Confidence", + "Domain", + "Evidence", + "EvidenceKind", + "Scope", + "ScopeGap", + "Support", + "UnsupportedClaim", + "by_domain", + "partially_verified", + "summary", + "support_for", + "unsupported_claims", +] diff --git a/tests/test_authority.py b/tests/test_authority.py new file mode 100644 index 0000000..2211ec9 --- /dev/null +++ b/tests/test_authority.py @@ -0,0 +1,435 @@ +"""Device-free tests for the authority layer. + +The tests worth writing here are the ones that try to get a model an authority it has not +earned. Three attacks matter more than the rest: handing a model a decision that requires an +interlock or a person, handing a plan a silence and having it read as permission, and +handing the ladder a passed simulation and having it read as a qualified machine. All three +produce a document that looks like governance, which is what makes them worth a test rather +than a paragraph. +""" + +from __future__ import annotations + +import pytest + +from autonomous_lab.authority import ( + DECISION_CLASSES, + DECISION_CLASSES_BY_NAME, + LADDER, + LADDER_BY_RUNG, + Assignment, + Authority, + DecisionClass, + Plan, + QualificationRung, + Rung, + ViolationKind, + implies, + missing_below, + qualified_for, + strengthened, + unaccountable, + unsupported_claims, + violations, +) + +# The keys of this repo's verified-literature index. Written out here rather than imported +# so that a typo or an invented citation in the registry fails a test instead of resolving +# to nothing. +KNOWN_EVIDENCE_KEYS = { + "a_lab", + "alabos", + "arrive", + "coley", + "dai", + "fda_di", + "homecage_multi", + "iso23783", + "jump", + "labsafety", + "ncats", + "provo", + "pylabrobot", + "roboculture", + "smartkage", +} + + +def _plan_assigning(**pairs) -> Plan: + """A plan covering every declared class, overridden where the test names one. + + Defaults to the required authority everywhere so that a test about one class is not + drowned in twelve unassigned-class violations. + """ + return Plan( + name="test", + assignments=tuple( + Assignment( + decision_class=dc.name, + authority=pairs.get(dc.name, dc.required), + who="a named person" if pairs.get(dc.name, dc.required) is Authority.HUMAN else "", + ) + for dc in DECISION_CLASSES + ), + ) + + +# -- the failure this module exists to catch ----------------------------------- + + +def test_a_model_may_not_release_a_batch(): + """The single most important test in the file. + + Release is where this lab's uncertainty becomes another party's assumption. A plan that + puts a model there must be reported, and reported as overreach rather than as a note. + """ + plan = _plan_assigning(batch_release=Authority.MODEL) + found = [v for v in violations(plan) if v.decision_class == "batch_release"] + assert len(found) == 1 + assert found[0].kind is ViolationKind.WEAKENED + assert found[0].required is Authority.HUMAN + assert found[0].assigned is Authority.MODEL + assert found[0].model_overreach + + +def test_a_model_may_not_arm_actuation(): + plan = _plan_assigning(actuation_arming=Authority.MODEL) + found = [v for v in violations(plan) if v.decision_class == "actuation_arming"] + assert len(found) == 1 + assert found[0].kind is ViolationKind.WEAKENED + assert found[0].model_overreach + + +def test_a_model_may_not_make_a_welfare_or_hazard_decision(): + plan = _plan_assigning( + welfare_intervention=Authority.MODEL, + hazard_call=Authority.MODEL, + animal_procedure_authorization=Authority.MODEL, + ) + names = {v.decision_class for v in violations(plan) if v.kind is ViolationKind.WEAKENED} + assert names == {"welfare_intervention", "hazard_call", "animal_procedure_authorization"} + + +def test_every_class_requiring_more_than_a_model_refuses_a_model(): + """Swept across the whole registry so a new class cannot be added without this holding.""" + for dc in DECISION_CLASSES: + if dc.required is Authority.MODEL: + continue + plan = _plan_assigning(**{dc.name: Authority.MODEL}) + weakened = [ + v + for v in violations(plan) + if v.decision_class == dc.name and v.kind is ViolationKind.WEAKENED + ] + assert weakened, f"{dc.name} requires {dc.required.value} and accepted a model" + + +def test_drafting_does_not_satisfy_an_interlock(): + """One step short is still a model in the path, which is what the requirement excludes.""" + plan = _plan_assigning(actuation_arming=Authority.MODEL_PROPOSES) + found = [v for v in violations(plan) if v.decision_class == "actuation_arming"] + assert found and found[0].kind is ViolationKind.WEAKENED + assert found[0].model_overreach + + +def test_a_deterministic_interlock_satisfies_a_deterministic_class(): + plan = _plan_assigning(actuation_arming=Authority.DETERMINISTIC) + assert not [v for v in violations(plan) if v.decision_class == "actuation_arming"] + + +# -- strengthening is never the defect ----------------------------------------- + + +def test_putting_a_human_on_a_model_permissible_class_is_not_a_violation(): + """A lab may always spend a person on a decision a model could make. + + The asymmetry is the whole design. If this ever fails, the check has become a rule about + conformity rather than a rule about safety. + """ + plan = _plan_assigning(prior_result_retrieval=Authority.HUMAN) + assert not [v for v in violations(plan) if v.decision_class == "prior_result_retrieval"] + assert plan.escalation_only + + +def test_strengthening_every_class_to_human_produces_no_violations(): + plan = Plan( + name="everything-to-a-person", + assignments=tuple( + Assignment(dc.name, Authority.HUMAN, who="the study director") for dc in DECISION_CLASSES + ), + ) + assert violations(plan) == () + assert plan.complies + + +def test_strengthening_is_reported_but_not_flagged(): + plan = _plan_assigning( + prior_result_retrieval=Authority.HUMAN, + actuation_arming=Authority.HUMAN, + ) + raised = {name for name, _req, _got in strengthened(plan)} + assert raised == {"prior_result_retrieval", "actuation_arming"} + assert not violations(plan) + + +def test_authority_ordering_runs_model_to_human(): + assert Authority.MODEL.rank < Authority.MODEL_PROPOSES.rank + assert Authority.MODEL_PROPOSES.rank < Authority.DETERMINISTIC.rank + assert Authority.DETERMINISTIC.rank < Authority.HUMAN.rank + assert Authority.HUMAN.satisfies(Authority.MODEL) + assert not Authority.MODEL.satisfies(Authority.HUMAN) + + +def test_only_one_authority_lets_a_model_decide_alone(): + """The load-bearing line on the enum. Everything else is a claim about review.""" + alone = [a for a in Authority if a.model_decides_alone] + assert alone == [Authority.MODEL] + in_path = {a for a in Authority if a.model_in_path} + assert in_path == {Authority.MODEL, Authority.MODEL_PROPOSES} + + +def test_no_hazard_or_welfare_class_admits_a_model_in_any_role(): + """Adding a reviewer to a model's framing of a hazard does not make it a hazard call.""" + for name in ("hazard_call", "welfare_intervention", "animal_procedure_authorization"): + assert not DECISION_CLASSES_BY_NAME[name].admits_a_model + + +# -- silence is not compliance -------------------------------------------------- + + +def test_an_unassigned_class_is_a_violation(): + """The defect every real authority document has: it covers what someone thought of.""" + plan = Plan( + name="two-lines", + assignments=( + Assignment("prior_result_retrieval", Authority.MODEL), + Assignment("protocol_drafting", Authority.MODEL_PROPOSES), + ), + ) + unassigned = {v.decision_class for v in violations(plan) if v.kind is ViolationKind.UNASSIGNED} + assert "batch_release" in unassigned + assert "actuation_arming" in unassigned + assert len(unassigned) == len(DECISION_CLASSES) - 2 + assert not plan.complies + + +def test_an_empty_plan_does_not_pass(): + plan = Plan(name="nothing") + found = violations(plan) + assert len(found) == len(DECISION_CLASSES) + assert all(v.kind is ViolationKind.UNASSIGNED for v in found) + + +def test_an_empty_plan_is_escalation_only_and_still_not_compliant(): + """The distinction that keeps escalation_only from being read as a pass. + + A plan weakening nothing is trivially escalation-only. It is also governing nothing, and + a property that could not tell those apart would be the most flattering number in the + package. + """ + plan = Plan(name="nothing") + assert plan.escalation_only + assert not plan.complies + + +def test_the_unassigned_reason_names_the_required_authority(): + """A finding a reader cannot act on is a finding they will not act on.""" + plan = Plan(name="nothing") + found = next(v for v in violations(plan) if v.decision_class == "batch_release") + assert "human" in found.reason + assert found.required is Authority.HUMAN + + +def test_an_assignment_to_an_undeclared_class_is_reported_not_ignored(): + plan = _plan_assigning() + plan = Plan( + name="with-a-stray", + assignments=plan.assignments + (Assignment("vibes_check", Authority.MODEL),), + ) + stray = [v for v in violations(plan) if v.kind is ViolationKind.UNDECLARED] + assert len(stray) == 1 + assert stray[0].decision_class == "vibes_check" + + +def test_a_human_class_with_nobody_named_is_reported_separately(): + """HUMAN authority with no name is a queue, not accountability. + + Deliberately not a violation: the authority level is right and only the accountability is + missing, so folding them together would let a fix for one look like a fix for the other. + """ + plan = Plan( + name="anonymous", + assignments=tuple( + Assignment(dc.name, dc.required, who="") for dc in DECISION_CLASSES + ), + ) + assert violations(plan) == () + missing = set(unaccountable(plan)) + assert "batch_release" in missing + assert "hazard_call" in missing + + +# -- the registry itself -------------------------------------------------------- + + +def test_every_decision_class_states_its_because(): + """A required authority with no mechanism cannot be argued with, only obeyed.""" + for dc in DECISION_CLASSES: + assert dc.because.strip(), f"{dc.name} declares a requirement with no mechanism" + assert len(dc.because) > 80, f"{dc.name} states a mechanism too thin to argue with" + + +def test_a_decision_class_cannot_be_built_without_a_mechanism(): + with pytest.raises(ValueError, match="no mechanism"): + DecisionClass( + name="bare", + decides="something", + required=Authority.HUMAN, + because=" ", + ) + + +def test_every_decision_class_says_what_is_being_decided(): + for dc in DECISION_CLASSES: + assert dc.decides.strip(), f"{dc.name} does not say what it decides" + + +def test_evidence_keys_resolve_to_the_verified_index(): + """An invented citation is worse than no citation, so an unknown key fails here.""" + for dc in DECISION_CLASSES: + if dc.evidence_key is not None: + assert dc.evidence_key in KNOWN_EVIDENCE_KEYS, f"{dc.name} cites an unknown source" + + +def test_the_registry_spans_every_authority_level(): + """A registry with nothing at MODEL would be posturing rather than an argument.""" + levels = {dc.required for dc in DECISION_CLASSES} + assert levels == set(Authority) + + +def test_class_names_are_unique(): + names = [dc.name for dc in DECISION_CLASSES] + assert len(names) == len(set(names)) + assert len(DECISION_CLASSES_BY_NAME) == len(names) + + +def test_the_hazard_argument_cites_the_safety_benchmark(): + assert DECISION_CLASSES_BY_NAME["hazard_call"].evidence_key == "labsafety" + assert DECISION_CLASSES_BY_NAME["hazard_call"].required is Authority.HUMAN + + +# -- the qualification ladder --------------------------------------------------- + + +def test_a_passed_simulation_does_not_qualify_the_physical_rungs(): + """The overclaim the ladder exists to refuse. + + A simulation establishes that the generated commands are the intended ones. Every rung + above it is a statement about matter, and there is no inference from one to the other. + """ + for rung in Rung: + if rung is Rung.SIMULATION: + continue + assert rung.physical + assert not implies(Rung.SIMULATION, rung) + assert implies(Rung.SIMULATION, Rung.SIMULATION) + + +def test_passing_rung_n_never_implies_rung_n_plus_one(): + order = list(Rung) + for lower, higher in zip(order, order[1:]): + assert not implies(lower, higher) + assert implies(higher, lower) + + +def test_a_rung_cannot_be_skipped(): + """Assay transfer on top of a simulation is qualified for the simulation. + + The upper result is not discarded; it simply stands on nothing, and a result about a + system nobody characterized is not a qualification of that system. + """ + passed = [Rung.SIMULATION, Rung.ASSAY_TRANSFER] + assert qualified_for(passed) == (Rung.SIMULATION,) + gaps = missing_below(Rung.ASSAY_TRANSFER, passed) + assert set(gaps) == { + Rung.DEVICE_ACCEPTANCE, + Rung.INTEGRATED_DRY_RUN, + Rung.VOLUMETRIC_QUALIFICATION, + } + + +def test_claiming_a_rung_never_passed_is_unsupported(): + unsupported = unsupported_claims([Rung.VOLUMETRIC_QUALIFICATION], [Rung.SIMULATION]) + assert len(unsupported) == 1 + claim, gaps = unsupported[0] + assert claim is Rung.VOLUMETRIC_QUALIFICATION + assert Rung.VOLUMETRIC_QUALIFICATION in gaps + assert Rung.DEVICE_ACCEPTANCE in gaps + + +def test_an_unbroken_ladder_qualifies_the_top(): + assert qualified_for(list(Rung)) == tuple(Rung) + assert unsupported_claims([Rung.SUSTAINED_PRODUCTION], list(Rung)) == () + + +def test_nothing_passed_qualifies_nothing(): + assert qualified_for([]) == () + assert missing_below(Rung.DEVICE_ACCEPTANCE, []) == (Rung.SIMULATION,) + + +def test_every_rung_states_what_it_does_not_qualify(): + """A rung stated without its limit will be read as qualifying everything above it.""" + assert len(LADDER) == len(Rung) + for entry in LADDER: + assert entry.qualifies.strip() + assert entry.does_not_qualify.strip(), f"{entry.rung.value} states no limit" + assert len(entry.does_not_qualify) > 80 + + +def test_a_rung_cannot_be_built_without_a_stated_limit(): + with pytest.raises(ValueError, match="no limit"): + QualificationRung( + rung=Rung.SIMULATION, + qualifies="everything, obviously", + does_not_qualify="", + ) + + +def test_the_ladder_is_ordered_and_complete(): + assert tuple(entry.rung for entry in LADDER) == tuple(Rung) + assert set(LADDER_BY_RUNG) == set(Rung) + heights = [entry.rung.height for entry in LADDER] + assert heights == sorted(heights) + + +def test_ladder_evidence_keys_resolve_to_the_verified_index(): + for entry in LADDER: + if entry.evidence_key is not None: + assert entry.evidence_key in KNOWN_EVIDENCE_KEYS + + +def test_the_assay_transfer_rung_refuses_to_qualify_production(): + """The one rung whose limit is stated by the cited guidance itself.""" + entry = LADDER_BY_RUNG[Rung.ASSAY_TRANSFER] + assert entry.evidence_key == "ncats" + assert "post-production monitoring" in entry.does_not_qualify + + +# -- no self-declaration -------------------------------------------------------- + + +def test_nothing_in_a_plan_can_declare_itself_compliant(): + """The same rule the ledger applies to automation and qc applies to readiness. + + There must be no field an author can set that suppresses a finding. If this fails, + someone has added an override and every report in this module can be made to lie. + """ + assignment = Assignment("batch_release", Authority.MODEL) + plan = Plan(name="p", assignments=(assignment,)) + for attr in ("approved", "exempt", "waived", "override", "compliant"): + assert not hasattr(assignment, attr) + assert not hasattr(plan, attr) + + # `complies` is computed from the registry and has no setter, so a plan cannot assert it. + with pytest.raises(AttributeError): + plan.complies = True diff --git a/tests/test_cadence.py b/tests/test_cadence.py new file mode 100644 index 0000000..01fc60d --- /dev/null +++ b/tests/test_cadence.py @@ -0,0 +1,490 @@ +"""Device-free tests for the cadence model. + +Three things are being defended here and they fail in different ways. + +The arithmetic is pinned exactly rather than approximately. 500 plates a day is 172.8 +seconds and nothing else, and a test that tolerated a percent of drift would let a rounding +bug through into the one number in this package that needs no evidence to be correct. + +The refusals have to survive being routed around. A target that does not say whether it +counts plates started, completed, QC-passed, or released must be refused whether it arrives +as a `Target` or as a bare float, because the convenient path is exactly where a caller in +a hurry will push. + +And `demonstrated_support` must never return a supporting citation. Not for a large target, +not for a small one, not for any of the four endpoints. The published record cited in this +module contains campaigns and capability demonstrations; it contains no sustained production +rate, and the moment a test tolerates one appearing, the module has started flattering the +field it exists to report on. +""" + +from __future__ import annotations + +import inspect + +import pytest + +from autonomous_lab.cadence import ( + DEMONSTRATIONS, + NO_INTEGRATED_PRODUCTION_RATE, + REFERENCES, + Cadence, + Demonstrated, + Endpoint, + Forcing, + Target, + cadence_for, + demonstrated_support, + headroom, + intervals, + launch_interval, + required_admissions, + sustained_rate_demonstrations, +) + + +# -- the arithmetic, pinned exactly -------------------------------------------- + + +def test_500_plates_a_day_is_a_plate_every_172_8_seconds(): + """86400 / 500. The whole module rests on this division being right.""" + assert launch_interval(500, 24) == 172.8 + + +def test_the_same_500_inside_a_staffed_day_is_115_2_seconds(): + """57600 / 500. Same target, no change in scope, 33 percent less time per plate.""" + assert launch_interval(500, 16) == 115.2 + + +def test_172_8_seconds_is_2_88_minutes_and_115_2_is_1_92(): + """The seconds are pinned exactly. Minutes are a display, so they get a tolerance.""" + c = cadence_for(Target(500, Endpoint.STARTED), operating_hours=16) + assert c.round_the_clock_seconds == 172.8 + assert c.interval_seconds == 115.2 + assert c.round_the_clock_seconds / 60.0 == pytest.approx(2.88) + assert c.minutes_between_plates == pytest.approx(1.92) + + +def test_both_intervals_are_reported_not_one(): + """Quoting only the 24-hour figure describes a lab that runs unattended.""" + assert intervals(500, 16) == (172.8, 115.2) + + +def test_the_walk_away_penalty_is_the_hours_ratio_and_nothing_else(): + """1.5x at 16 hours, and it does not depend on the target, protocol, or instruments.""" + c16 = cadence_for(Target(500, Endpoint.STARTED), operating_hours=16) + c8 = cadence_for(Target(37, Endpoint.STARTED), operating_hours=8) + assert c16.walk_away_multiplier == 1.5 + assert c8.walk_away_multiplier == 3.0 + + +def test_a_window_longer_than_the_day_is_refused(): + """A 30-hour window makes any target look achievable by arithmetic alone.""" + with pytest.raises(ValueError, match="does not fit in a day"): + launch_interval(500, 30) + + +def test_a_zero_or_negative_window_is_refused(): + with pytest.raises(ValueError): + launch_interval(500, 0) + with pytest.raises(ValueError): + launch_interval(500, -8) + + +def test_a_zero_or_negative_target_is_not_a_target(): + with pytest.raises(ValueError): + Target(0, Endpoint.STARTED) + with pytest.raises(ValueError): + Target(-1, Endpoint.STARTED) + with pytest.raises(ValueError): + launch_interval(0, 24) + + +# -- the endpoint refusal ------------------------------------------------------ + + +def test_a_target_that_does_not_say_what_it_counts_is_refused(): + """Not answered with the loosest reading. Refused.""" + with pytest.raises(ValueError) as err: + cadence_for(Target(500)) + assert "does not say what it counts" in str(err.value) + + +def test_the_refusal_names_all_four_endpoints_so_the_caller_can_pick_one(): + with pytest.raises(ValueError) as err: + cadence_for(Target(500)) + message = str(err.value) + for word in ("STARTED", "COMPLETED", "QC_PASSED", "RELEASED"): + assert word in message + + +def test_a_bare_number_cannot_route_around_the_refusal(): + """The convenient path is where a caller in a hurry pushes, so it is closed too.""" + with pytest.raises(ValueError, match="does not say what it counts"): + cadence_for(500) + + +def test_demonstrated_support_refuses_an_unstated_endpoint_rather_than_answering(): + support = demonstrated_support(500) + assert support.refused + assert not support.supported + assert support.nearest == () + + +def test_a_refused_target_gets_no_citations_at_all(): + """Listing the nearest evidence before the question is well posed invites cherry-picking.""" + support = demonstrated_support(Target(500)) + assert support.refused + assert support.supporting() == () + assert len(support.nearest) == 0 + + +def test_only_started_is_an_admission_rate(): + assert Endpoint.STARTED.is_admission_rate + assert not Endpoint.COMPLETED.is_admission_rate + assert not Endpoint.QC_PASSED.is_admission_rate + assert not Endpoint.RELEASED.is_admission_rate + + +def test_a_downstream_target_has_no_admission_rate_without_a_measured_yield(): + """None, not an assumed yield of 1.0. Assuming 1.0 under-admits by exactly the failure rate.""" + assert required_admissions(Target(500, Endpoint.RELEASED)) is None + assert required_admissions(Target(500, Endpoint.QC_PASSED)) is None + assert required_admissions(Target(500, Endpoint.COMPLETED)) is None + + +def test_a_started_target_needs_no_yield_and_ignores_one_offered(): + assert required_admissions(Target(500, Endpoint.STARTED)) == 500 + assert required_admissions(Target(500, Endpoint.STARTED), yield_fraction=0.5) == 500 + + +def test_a_rerun_rate_raises_the_admission_rate_and_shortens_the_interval(): + """500 released at 80 percent yield is 625 admitted, and 138.24 s rather than 172.8 s.""" + admissions = required_admissions(Target(500, Endpoint.RELEASED), yield_fraction=0.8) + assert admissions == 625 + assert launch_interval(admissions, 24) == 138.24 + assert launch_interval(admissions, 24) < launch_interval(500, 24) + + +def test_required_admissions_refuses_an_ambiguous_target_and_a_non_fraction_yield(): + with pytest.raises(ValueError, match="does not say what it counts"): + required_admissions(Target(500)) + with pytest.raises(ValueError): + required_admissions(Target(500, Endpoint.RELEASED), yield_fraction=0) + with pytest.raises(ValueError): + required_admissions(Target(500, Endpoint.RELEASED), yield_fraction=1.4) + + +# -- what the interval forces -------------------------------------------------- + + +def test_a_target_below_the_slowest_declared_step_is_unachievable_on_one_line(): + """The step holds the line for 600 s and a plate must enter every 172.8 s.""" + c = cadence_for(Target(500, Endpoint.STARTED), slowest_step_seconds=600) + assert c.single_line_feasible is False + assert c.lines_required == 4 + lines = _forcing(c, Forcing.LINES) + assert lines.unachievable + assert "unachievable on a single line" in lines.statement + + +def test_the_unachievable_finding_says_other_instruments_do_not_help(): + """'Just make the rest faster' is the reflex, and the arithmetic says it changes nothing.""" + c = cadence_for(Target(500, Endpoint.STARTED), slowest_step_seconds=600) + statement = _forcing(c, Forcing.LINES).statement + assert "Speeding up every OTHER instrument changes nothing" in statement + + +def test_the_line_count_rounds_up_because_three_and_a_half_lines_do_not_exist(): + c = cadence_for(Target(500, Endpoint.STARTED), slowest_step_seconds=600) + assert 600 / c.interval_seconds > 3 + assert c.lines_required == 4 + + +def test_a_step_inside_the_interval_needs_one_line_and_says_what_that_does_not_prove(): + c = cadence_for(Target(500, Endpoint.STARTED), slowest_step_seconds=60) + assert c.single_line_feasible is True + assert c.lines_required == 1 + lines = _forcing(c, Forcing.LINES) + assert not lines.unachievable + assert "assumes the caller named the true slowest step" in lines.statement + + +def test_an_undeclared_slowest_step_is_refused_rather_than_defaulted(): + """Defaulting to zero would report one line as sufficient for every target ever stated.""" + c = cadence_for(Target(500, Endpoint.STARTED)) + assert c.lines_required is None + assert c.single_line_feasible is None + lines = _forcing(c, Forcing.LINES) + assert not lines.computable + assert lines.value is None + assert "not computable" in lines.statement + + +def test_buffer_is_refused_without_an_end_to_end_duration(): + c = cadence_for(Target(500, Endpoint.STARTED)) + assert c.plates_in_flight is None + buffer = _forcing(c, Forcing.BUFFER) + assert not buffer.computable + assert buffer.value is None + + +def test_buffer_counts_the_plates_that_must_be_physically_resident(): + """A 2-hour end-to-end at a 172.8 s interval means 42 plates are in the workcell at once.""" + c = cadence_for(Target(500, Endpoint.STARTED), end_to_end_seconds=7200) + assert c.plates_in_flight == 42 + buffer = _forcing(c, Forcing.BUFFER) + assert buffer.computable + assert buffer.value == 42.0 + + +def test_recovery_and_upkeep_need_no_declared_duration_at_all(): + """Both are arithmetic on the interval, so neither can be refused for missing data.""" + c = cadence_for(Target(500, Endpoint.STARTED)) + assert _forcing(c, Forcing.RECOVERY).computable + assert _forcing(c, Forcing.UPKEEP).computable + + +def test_a_stoppage_costs_admissions_at_exactly_the_launch_rate(): + """An hour down at 500/day costs 20.8 plates, and the day has no slack to make them back.""" + c = cadence_for(Target(500, Endpoint.STARTED)) + assert c.admissions_lost(3600) == 3600 / 172.8 + assert c.admissions_lost(0) == 0.0 + with pytest.raises(ValueError): + c.admissions_lost(-1) + + +def test_a_short_stop_is_not_rounded_down_to_free(): + c = cadence_for(Target(500, Endpoint.STARTED)) + assert 0 < c.admissions_lost(60) < 1 + + +def test_upkeep_shortens_the_interval_because_it_shares_the_window(): + """An hour of daily upkeep turns 172.8 s into 165.6 s for the same target.""" + c = cadence_for(Target(500, Endpoint.STARTED)) + assert c.interval_net_of(3600) == 165.6 + assert c.interval_net_of(0) == c.interval_seconds + + +def test_upkeep_that_consumes_the_window_is_refused_not_returned_as_a_number(): + """A negative interval formats as a number and would be quoted as one.""" + c = cadence_for(Target(500, Endpoint.STARTED)) + with pytest.raises(ValueError, match="consumes the whole"): + c.interval_net_of(86400) + with pytest.raises(ValueError): + c.interval_net_of(90000) + + +def test_every_forcing_appears_exactly_once_including_the_refused_ones(): + """An omitted constraint reads as inapplicable, which is not what a missing duration means.""" + c = cadence_for(Target(500, Endpoint.STARTED)) + seen = [i.forcing for i in c.implied_constraints()] + assert seen == [Forcing.LINES, Forcing.BUFFER, Forcing.RECOVERY, Forcing.UPKEEP] + assert len(set(seen)) == len(Forcing) + + +def test_only_lines_and_buffer_are_relieved_by_hardware(): + """Two of the four are purchases. The other two are charges against the same clock.""" + assert Forcing.LINES.relieved_by_hardware + assert Forcing.BUFFER.relieved_by_hardware + assert not Forcing.RECOVERY.relieved_by_hardware + assert not Forcing.UPKEEP.relieved_by_hardware + + +def test_every_implication_shows_its_arithmetic(): + c = cadence_for(Target(500, Endpoint.STARTED), slowest_step_seconds=600, end_to_end_seconds=7200) + for imp in c.implied_constraints(): + assert imp.arithmetic + assert imp.statement + + +def test_a_declared_duration_of_zero_or_less_is_refused(): + with pytest.raises(ValueError, match="not a duration"): + cadence_for(Target(500, Endpoint.STARTED), slowest_step_seconds=0) + with pytest.raises(ValueError, match="not a duration"): + cadence_for(Target(500, Endpoint.STARTED), end_to_end_seconds=-5) + + +# -- the human share of the clock ---------------------------------------------- + + +def test_an_intervention_rate_high_enough_to_consume_the_clock_makes_the_target_unreachable(): + """A human on half of all plates for 10 minutes is 300 s against a 172.8 s interval.""" + h = headroom(172.8, intervention_rate=0.5, minutes_per_intervention=10) + assert h.human_seconds_per_plate == 300.0 + assert h.consumed_fraction > 1.0 + assert not h.reachable + assert h.slack_seconds < 0 + assert h.operators_required == 2 + + +def test_exactly_one_hundred_percent_of_the_clock_is_not_reachable(): + """Zero slack plus any variance is a backlog that grows rather than one that hovers.""" + h = headroom(172.8, intervention_rate=1.0, minutes_per_intervention=172.8 / 60.0) + assert h.consumed_fraction == 1.0 + assert h.slack_seconds == 0.0 + assert not h.reachable + + +def test_a_survivable_intervention_rate_reports_what_is_left_and_what_still_wants_it(): + h = headroom(172.8, intervention_rate=0.1, minutes_per_intervention=2) + assert h.human_seconds_per_plate == 12.0 + assert h.reachable + assert h.slack_seconds == 160.8 + assert "upkeep and the recovery" in h.why() + + +def test_no_interventions_leaves_the_whole_clock_and_names_no_operator(): + h = headroom(172.8, intervention_rate=0.0, minutes_per_intervention=30) + assert h.consumed_fraction == 0.0 + assert h.reachable + assert h.operators_required == 0 + assert h.ceiling_interval_seconds is None + assert h.ceiling_plates_per_day() is None + + +def test_the_finding_says_instrument_speed_is_not_in_the_arithmetic(): + """This is the sentence the module exists to be able to print.""" + h = headroom(172.8, intervention_rate=0.5, minutes_per_intervention=10) + why = h.why() + assert "Instrument speed appears nowhere in this arithmetic" in why + assert "faster robot does not move it" in why + + +def test_headroom_takes_no_instrument_argument_of_any_kind(): + """Structural, not textual: there is no parameter a faster machine could be declared in.""" + params = list(inspect.signature(headroom).parameters) + assert params == ["interval_seconds", "intervention_rate", "minutes_per_intervention"] + + +def test_only_the_product_of_rate_and_duration_matters(): + """Rare-and-long and frequent-and-short cost the identical share of the clock.""" + rare = headroom(172.8, intervention_rate=0.5, minutes_per_intervention=10) + frequent = headroom(172.8, intervention_rate=1.0, minutes_per_intervention=5) + assert rare.human_seconds_per_plate == frequent.human_seconds_per_plate + assert rare.consumed_fraction == frequent.consumed_fraction + + +def test_one_operator_has_a_plates_per_day_ceiling_that_ignores_the_robot_entirely(): + """5 minutes of attention per plate caps a 24-hour day at 288 plates, whatever is installed.""" + h = headroom(172.8, intervention_rate=1.0, minutes_per_intervention=5) + assert h.ceiling_interval_seconds == 300.0 + assert h.ceiling_plates_per_day(24) == 288.0 + assert h.ceiling_plates_per_day(8) == 96.0 + + +def test_headroom_refuses_impossible_inputs(): + with pytest.raises(ValueError): + headroom(0, 0.1, 2) + with pytest.raises(ValueError): + headroom(172.8, -0.1, 2) + with pytest.raises(ValueError): + headroom(172.8, 0.1, -2) + + +# -- the honest check ---------------------------------------------------------- + + +def test_demonstrated_support_never_returns_a_supporting_citation(): + """For any endpoint, at any target. This is the assertion the module is built around.""" + for endpoint in Endpoint: + support = demonstrated_support(Target(500, endpoint)) + assert not support.refused + assert not support.supported + assert support.supporting() == () + + +def test_lowering_the_target_does_not_manufacture_support(): + """A cited campaign is not a rate, so shrinking the number does not turn one into one.""" + for plates in (1, 6, 500, 50_000): + support = demonstrated_support(Target(plates, Endpoint.STARTED)) + assert not support.supported + assert support.supporting() == () + + +def test_no_cited_work_claims_a_sustained_rate(): + assert sustained_rate_demonstrations() == () + assert all(d.demonstrated is not Demonstrated.SUSTAINED_RATE for d in DEMONSTRATIONS) + assert all(not d.establishes_a_rate for d in DEMONSTRATIONS) + + +def test_the_reason_states_that_no_cited_system_establishes_it(): + support = demonstrated_support(Target(500, Endpoint.RELEASED)) + assert NO_INTEGRATED_PRODUCTION_RATE in support.reason + assert "No cited published system establishes integrated multi-domain" in support.reason + + +def test_the_support_flag_is_computed_from_the_table_rather_than_hard_coded(): + """If a genuine sustained-rate work is ever added, this flips without editing the flag.""" + support = demonstrated_support(Target(500, Endpoint.STARTED)) + assert support.supported == bool(support.supporting()) + assert support.nearest == DEMONSTRATIONS + + +def test_every_demonstration_records_what_it_does_not_establish(): + """A citation table whose caveats are optional degrades into a list of names.""" + for d in DEMONSTRATIONS: + assert d.does_not_establish.strip() + assert len(d.does_not_establish) > 100 + assert d.ran_for.strip() + assert d.scale.strip() + + +def test_every_demonstration_resolves_to_a_full_citation(): + for d in DEMONSTRATIONS: + assert d.key in REFERENCES + assert d.citation == REFERENCES[d.key] + + +def test_the_only_continuous_multi_site_work_in_the_table_actuates_nothing(): + """The longest-running system cited does not run experiments. It watches cages.""" + keys = [d.key for d in DEMONSTRATIONS if not d.actuates] + assert "homecage_multi" in keys + watcher = next(d for d in DEMONSTRATIONS if d.key == "homecage_multi") + assert watcher.demonstrated is Demonstrated.CAMPAIGN + assert "It observes" in watcher.does_not_establish + assert "retrospective" in watcher.does_not_establish + + +def test_the_only_per_day_figure_in_the_table_is_flagged_as_a_peak(): + """A peak day is the opposite of a sustained cadence, and it is the only number available.""" + alabos = next(d for d in DEMONSTRATIONS if d.key == "alabos") + assert "peak" in alabos.scale + assert "PEAK day" in alabos.does_not_establish + + +def test_a_published_plates_per_day_number_exists_and_is_a_study_design(): + """3 or 6 plates a day, from pre-production validation guidance, not an achieved throughput.""" + ncats = next(d for d in DEMONSTRATIONS if d.key == "ncats") + assert ncats.demonstrated is Demonstrated.NO_RATE_CLAIM + assert "STUDY DESIGN" in ncats.does_not_establish + + +def test_the_authors_own_throughput_disclaimers_are_recorded_not_paraphrased_away(): + roboculture = next(d for d in DEMONSTRATIONS if d.key == "roboculture") + assert "explicitly disclaim throughput" in roboculture.does_not_establish + smartkage = next(d for d in DEMONSTRATIONS if d.key == "smartkage") + assert "low throughput" in smartkage.does_not_establish + + +# -- shape --------------------------------------------------------------------- + + +def test_a_cadence_is_frozen_so_a_reported_interval_cannot_be_edited_afterwards(): + c = cadence_for(Target(500, Endpoint.STARTED)) + assert isinstance(c, Cadence) + with pytest.raises(Exception): + c.interval_seconds = 1.0 # type: ignore[misc] + + +def test_the_constraints_recompute_rather_than_being_stored(): + """Two calls agree because both derive from the interval, not from a cached list.""" + c = cadence_for(Target(500, Endpoint.STARTED), slowest_step_seconds=600) + first = c.implied_constraints() + second = c.implied_constraints() + assert [i.forcing for i in first] == [i.forcing for i in second] + assert [i.value for i in first] == [i.value for i in second] + + +def _forcing(cadence: Cadence, forcing: Forcing): + return next(i for i in cadence.implied_constraints() if i.forcing is forcing) diff --git a/tests/test_literature.py b/tests/test_literature.py new file mode 100644 index 0000000..79463cb --- /dev/null +++ b/tests/test_literature.py @@ -0,0 +1,437 @@ +"""Device-free tests for the citation layer. Every one of them tries to make it lie. + +The failure this module exists to prevent is an accurate citation carrying an inaccurate +inference, so the tests are written from the attacker's side: hand it a cell-assay production +claim and see whether a materials campaign comes back as support, hand it a claim no work +addresses and see whether an empty list reports as satisfied, and check that an entry whose +numbers were only partly confirmed cannot present itself as fully verified. + +Two invariants carry the rest. Every entry must state what it does NOT establish -- a citation +with no boundary is an unread one, and the table is worth nothing if it can hold one. And every +number that failed verification must still be written down somewhere in the entry, because a +verification pass whose misses are invisible only launders the numbers it could not check. + +One test deliberately proves the layer can say yes. A checker that refuses everything is as +useless as one that accepts everything, and the refusals only mean something if a scope that +genuinely sits inside a demonstration comes back supported. +""" + +from __future__ import annotations + +import pytest + +from autonomous_lab.literature import ( + EVIDENCE, + EVIDENCE_BY_KEY, + UNSUPPORTED, + Confidence, + Domain, + Evidence, + EvidenceKind, + Scope, + Support, + by_domain, + partially_verified, + summary, + support_for, + unsupported_claims, +) + + +# A claim of the kind this module was built to refuse: sustained, unattended, cell-assay +# production. Every field is what gets asserted in the room, not what anyone has shown. +SUSTAINED_CELL_PRODUCTION = Scope( + summary="continuous unattended plate-based cell assay production", + material="mammalian cell culture in plates", + continuous_hours=720.0, + replicated=True, + sustained=True, + unattended=True, +) + + +def _gap(support, key: str): + """The in-domain finding for one citation. + + Raises rather than returning None if the citation was cut on domain instead, so a test that + meant to check a scope shortfall cannot quietly pass because the entry never reached the + comparison at all. + """ + return next(g for g in support.speaks_to if g.evidence.key == key) + + +def _refused(support, key: str): + """The domain-cut finding for one citation. Kept separate from `_gap` because the two + lists mean different things: one is a citation that fell short, the other is a citation + about something else.""" + return next(g for g in support.out_of_domain if g.evidence.key == key) + + +# -- the domain cut ------------------------------------------------------------ + + +def test_a_materials_campaign_is_not_support_for_a_cell_assay_production_claim(): + """The single most common miscitation in this field, and the reason for the module.""" + support = support_for(Domain.CELL_ASSAY, SUSTAINED_CELL_PRODUCTION) + assert "a_lab" not in {g.evidence.key for g in support.speaks_to} + assert "a_lab" in {g.evidence.key for g in support.out_of_domain} + + +def test_the_materials_campaign_is_refused_in_the_words_that_matter(): + """'Not supported' is an opinion. Naming the scope is a statement someone can check.""" + support = support_for(Domain.CELL_ASSAY, SUSTAINED_CELL_PRODUCTION) + gap = _refused(support, "a_lab") + assert "17-day materials campaign" in gap.reason + assert "does not speak to a cell assay claim" in gap.reason + + +def test_a_fifteen_hour_yeast_run_does_not_reach_sustained_production(): + """In-domain and still short. This is the citation that survives the domain cut.""" + support = support_for(Domain.CELL_ASSAY, SUSTAINED_CELL_PRODUCTION) + gap = _gap(support, "roboculture") + assert not gap.covers + assert "15-hour single-plate yeast run" in gap.reason + assert "does not reach sustained production" in gap.reason + + +def test_crossing_into_living_material_is_named_rather_than_left_to_the_reader(): + support = support_for(Domain.CELL_ASSAY, SUSTAINED_CELL_PRODUCTION) + gap = _refused(support, "dai") + assert "loses viability" in gap.reason + + +def test_the_living_line_falls_where_the_failure_modes_do(): + assert Domain.CELL_ASSAY.living and Domain.IN_VIVO.living + assert not Domain.MATERIALS.living + assert not Domain.CHEMISTRY.living + assert not Domain.ORCHESTRATION.living + + +def test_support_for_accounts_for_every_entry_exactly_once(): + """A citation the report drops is a citation nobody has to answer for.""" + support = support_for(Domain.CELL_ASSAY, SUSTAINED_CELL_PRODUCTION) + seen = [g.evidence.key for g in support.speaks_to] + [g.evidence.key for g in support.out_of_domain] + assert sorted(seen) == sorted(e.key for e in EVIDENCE) + + +def test_the_genomics_style_claim_is_unsupported_and_says_why(): + support = support_for(Domain.CELL_ASSAY, SUSTAINED_CELL_PRODUCTION) + assert not support.supported + assert support.gaps() + assert "sustained production" in support.reason() + + +# -- the layer must be able to say yes, or its refusals mean nothing ----------- + + +def test_a_scope_that_sits_inside_a_demonstration_reports_as_supported(): + """Guards against the cheapest possible bug: a checker that always answers no.""" + modest = Scope( + summary="a single ten-hour attended run on one plate of yeast", + material="yeast culture in a 96-well plate", + continuous_hours=10.0, + ) + support = support_for(Domain.CELL_ASSAY, modest) + assert support.supported + covering = {g.evidence.key for g in support.speaks_to if g.covers} + assert "roboculture" in covering + + +def test_the_same_demonstration_fails_the_same_claim_once_unattended_is_asserted(): + """One field changes, and the citation that covered the claim stops covering it.""" + attended = Scope(summary="ten attended hours", continuous_hours=10.0) + unattended = Scope(summary="ten unattended hours", continuous_hours=10.0, unattended=True) + before = _gap(support_for(Domain.CELL_ASSAY, attended), "roboculture") + after = _gap(support_for(Domain.CELL_ASSAY, unattended), "roboculture") + assert before.covers + assert not after.covers + assert "manually initiated" in after.reason + + +def test_an_empty_support_list_is_not_vacuously_satisfied(): + """all() over an empty sequence is True, and that bug would rank the worst claims best.""" + empty = Support( + claim_domain=Domain.CELL_ASSAY, + scope=SUSTAINED_CELL_PRODUCTION, + speaks_to=(), + out_of_domain=(), + ) + assert not empty.supported + + +# -- kind: what can evidence an operating claim at all ------------------------ + + +def test_only_a_demonstration_evidences_an_operating_claim(): + assert EvidenceKind.EMPIRICAL_DEMONSTRATION.demonstrates_operation + for kind in ( + EvidenceKind.BENCHMARK, + EvidenceKind.GUIDANCE, + EvidenceKind.STANDARD, + EvidenceKind.RESOURCE, + ): + assert not kind.demonstrates_operation + + +def test_in_domain_guidance_does_not_cover_an_operating_claim(): + """A standard's existence is easy to mistake for its satisfaction.""" + support = support_for(Domain.CELL_ASSAY, SUSTAINED_CELL_PRODUCTION) + gap = _gap(support, "ncats") + assert not gap.covers + assert "not a demonstration" in gap.reason + + +def test_a_benchmark_is_not_a_lab_run(): + bench_claim = Scope(summary="hazard judgment at a bench", unattended=True) + gap = _gap(support_for(Domain.SAFETY, bench_claim), "labsafety") + assert not gap.covers + assert "does not show a lab operating" in gap.reason + + +def test_a_non_demonstration_is_refused_once_rather_than_four_times(): + """Duration, replication and human-in-the-loop gaps against a checklist are category errors. + + A report that stacked them would read as four objections, three of them fabricated, and the + real one -- nobody ran anything -- would be the easiest of the four to skim past. + """ + support = support_for(Domain.CELL_ASSAY, SUSTAINED_CELL_PRODUCTION) + for key in ("ncats", "jump"): + gap = _gap(support, key) + assert len(gap.shortfalls) == 1, f"{key} is refused on {len(gap.shortfalls)} grounds" + assert "sustained production" not in gap.reason + assert "was not replicated" not in gap.reason + + +# -- confidence ---------------------------------------------------------------- + + +def test_a_work_whose_numbers_were_only_partially_verified_is_not_confirmed(): + partial = partially_verified() + assert partial, "a verification pass that confirmed everything did not check hard enough" + for e in partial: + assert e.confidence is Confidence.PARTIAL + assert not e.confidence.fully_verified + assert not e.verified + + +def test_every_partial_entry_names_which_numbers_could_not_be_confirmed(): + """A confidence downgrade with no reason attached cannot be acted on or retired.""" + for e in partially_verified(): + assert e.unverified + for item in e.unverified: + assert len(item.strip()) > 20, f"{e.key} carries an unverified item too vague to chase" + + +def test_a_partial_entry_that_names_nothing_unverified_is_refused(): + with pytest.raises(ValueError, match="names nothing unverified"): + Evidence( + key="hollow", + citation="A work.", + url="https://example.org/1", + domain=Domain.CELL_ASSAY, + kind=EvidenceKind.EMPIRICAL_DEMONSTRATION, + year=2020, + confidence=Confidence.PARTIAL, + shorthand="a work", + demonstrates="something", + does_not_establish="it does not establish anything else", + scope=Scope(summary="a scope"), + ) + + +def test_confirmed_entries_still_carry_the_numbers_that_failed_checking(): + """Confirmed means the stated numbers held, not that nothing went unchecked.""" + confirmed_with_misses = [e for e in EVIDENCE if e.verified and e.unverified] + assert confirmed_with_misses + assert "a_lab" in {e.key for e in confirmed_with_misses} + + +def test_the_figures_that_failed_verification_are_stated_nowhere_as_fact(): + """Dropping them would be safe. Restating them would not. The table does neither.""" + a_lab = EVIDENCE_BY_KEY["a_lab"] + assert "43 new materials" not in a_lab.demonstrates + assert "41 of 58" not in a_lab.demonstrates + assert "43 new materials" in " ".join(a_lab.unverified) + + coley = EVIDENCE_BY_KEY["coley"] + assert "68 hours" not in coley.demonstrates + assert "68 hours" in " ".join(coley.unverified) + + +def test_coley_does_not_attribute_safety_to_the_expert_step(): + """The one attribution that could not be confirmed, and the reason this entry is PARTIAL.""" + coley = EVIDENCE_BY_KEY["coley"] + assert coley.confidence is Confidence.PARTIAL + assert "safety" not in coley.demonstrates.lower() + assert "safety" in " ".join(coley.unverified).lower() + + +def test_pylabrobot_does_not_claim_simulation_across_families(): + """The paper asserts it architecturally and never reports running it. So does this entry.""" + plr = EVIDENCE_BY_KEY["pylabrobot"] + assert plr.confidence is Confidence.PARTIAL + assert "designed to work with any robot" in " ".join(plr.unverified) + + +# -- the boundary field -------------------------------------------------------- + + +def test_every_entry_states_what_it_does_not_establish(): + """A citation with no stated scope limit is the failure mode this module prevents.""" + for e in EVIDENCE: + assert e.does_not_establish.strip(), f"{e.key} states no scope limit" + assert len(e.does_not_establish) > 200, f"{e.key} states a boundary too thin to be real" + + +def test_no_two_entries_share_a_boundary(): + """Boilerplate pasted across entries would satisfy the length check and say nothing.""" + boundaries = [e.does_not_establish for e in EVIDENCE] + assert len(set(boundaries)) == len(boundaries) + + +def test_an_entry_with_no_boundary_is_refused_at_construction(): + with pytest.raises(ValueError, match="states no scope limit"): + Evidence( + key="unbounded", + citation="A famous paper.", + url="https://doi.org/10.0000/fake", + domain=Domain.MATERIALS, + kind=EvidenceKind.EMPIRICAL_DEMONSTRATION, + year=2024, + confidence=Confidence.CONFIRMED, + shorthand="a famous paper", + demonstrates="a great deal", + does_not_establish=" ", + scope=Scope(summary="a scope"), + ) + + +# -- citation hygiene ---------------------------------------------------------- + + +def test_every_entry_carries_a_url_or_doi(): + for e in EVIDENCE: + assert e.url.startswith("http"), f"{e.key} carries no resolvable identifier" + assert e.citation.strip(), f"{e.key} carries no citation" + + +def test_an_entry_with_no_url_is_refused_at_construction(): + with pytest.raises(ValueError, match="no URL or DOI"): + Evidence( + key="uncheckable", + citation="Someone, somewhere.", + url="", + domain=Domain.CHEMISTRY, + kind=EvidenceKind.EMPIRICAL_DEMONSTRATION, + year=2021, + confidence=Confidence.CONFIRMED, + shorthand="a work nobody can look up", + demonstrates="something", + does_not_establish="it does not establish that it can be found", + scope=Scope(summary="a scope"), + ) + + +def test_evidence_keys_are_unique_and_the_index_is_complete(): + keys = [e.key for e in EVIDENCE] + assert len(set(keys)) == len(keys) + assert set(EVIDENCE_BY_KEY) == set(keys) + + +def test_every_declared_domain_has_at_least_one_entry(): + """An empty domain would silently return no support instead of refusing a claim.""" + for domain in Domain: + assert by_domain(domain), f"no cited work for domain '{domain.value}'" + + +# -- scope honesty ------------------------------------------------------------- + + +def test_a_hedged_duration_is_not_converted_into_a_figure(): + """'Almost four consecutive days' is not 96 hours, and the table refuses to make it one.""" + dai = EVIDENCE_BY_KEY["dai"] + assert dai.scope.continuous_hours is None + assert "almost four consecutive days" in dai.scope.duration_verbatim + + +def test_a_hedged_duration_reports_as_a_gap_rather_than_passing_quietly(): + claim = Scope(summary="a week of continuous chemistry", continuous_hours=168.0) + gap = _gap(support_for(Domain.CHEMISTRY, claim), "dai") + assert not gap.covers + assert "states no duration that converts to a figure" in gap.reason + + +def test_no_demonstration_in_this_table_ran_without_a_human(): + """The finding, stated as an invariant: every one of them kept somebody in the loop.""" + demos = [e for e in EVIDENCE if e.kind.demonstrates_operation] + assert demos + for e in demos: + assert e.scope.human_in_loop.strip(), f"{e.key} claims fully unattended operation" + + +def test_no_work_in_this_table_demonstrates_sustained_production(): + for e in EVIDENCE: + assert not e.scope.sustained, f"{e.key} claims sustained production" + + +def test_a_shorter_run_cannot_cover_a_longer_claim(): + claim = Scope(summary="a hundred continuous hours of cell work", continuous_hours=100.0) + gap = _gap(support_for(Domain.CELL_ASSAY, claim), "roboculture") + assert not gap.covers + assert "ran 15 h against the 100 h" in gap.reason + + +# -- the standing list --------------------------------------------------------- + + +def test_the_unsupported_list_is_not_empty_and_names_the_production_rate_claim(): + claims = unsupported_claims() + assert claims + names = {c.name for c in claims} + assert "integrated_autonomy_at_production_rates" in names + + +def test_the_unsupported_list_covers_the_five_recurring_inferences(): + names = {c.name for c in UNSUPPORTED} + assert names == { + "integrated_autonomy_at_production_rates", + "nominal_speed_as_biological_throughput", + "simulation_as_qualification", + "ai_fluency_as_hazard_competence", + "capital_as_capacity", + } + + +def test_every_unsupported_claim_names_the_citations_usually_offered_for_it(): + """The argument is never 'you have no evidence'; it is where the evidence stops.""" + for c in UNSUPPORTED: + assert c.nearest, f"{c.name} refuses a claim without naming what it refuses" + for key in c.nearest: + assert key in EVIDENCE_BY_KEY, f"{c.name} cites '{key}', which is not in the table" + + +def test_every_unsupported_claim_says_what_would_retire_it(): + """An objection with no stated remedy is a position, not a finding.""" + for c in UNSUPPORTED: + assert len(c.would_establish_it.strip()) > 40, f"{c.name} states no way to retire itself" + assert len(c.why_unsupported.strip()) > 200, f"{c.name} asserts rather than argues" + + +def test_the_production_rate_claim_argues_from_the_specific_shortfalls(): + c = next(c for c in UNSUPPORTED if c.name == "integrated_autonomy_at_production_rates") + assert "17-day" in c.why_unsupported + assert "15 hours" in c.why_unsupported + assert set(c.nearest) >= {"a_lab", "roboculture"} + + +# -- the report ---------------------------------------------------------------- + + +def test_summary_adds_up(): + s = summary() + assert s["evidence"] == len(EVIDENCE) + assert s["confirmed"] + s["partial"] == s["evidence"] + assert s["demonstrations"] < s["evidence"], "not every citation is a lab that ran" + assert s["domains_covered"] == len(Domain) + assert s["unsupported_claims"] == len(UNSUPPORTED) + assert s["entries_with_unverified_numbers"] >= s["partial"] From 75b1fd2403dc76835981cd4dfb5518068a577e92 Mon Sep 17 00:00:00 2001 From: di-omics Date: Tue, 28 Jul 2026 02:43:57 -0700 Subject: [PATCH 2/3] Fix what an adversarial audit found in the layer built to prevent overclaiming An independent audit of these three modules returned eleven blockers. The irony is the useful part: the module written to stop autonomy claims outrunning their evidence was doing it itself, in three separate ways. literature never read Scope.material. `support_for` on a primary human hepatocyte claim returned supported=True, citing a fifteen-hour yeast run whose own does_not_establish says it establishes nothing about mammalian culture. Material is now compared first, before any duration or attendance check, because a duration gap on the wrong organism is a rounding error next to the organism. Comparison is exact-match and refuses on any difference, since deciding that two material descriptions are equivalent IS the transfer question and a fuzzy match would answer it silently in the direction of support. The same function treated an unwritten claim as a satisfied one. Every comparison was gated on the claim asserting something, so a Scope naming no material, duration, replication or attendance produced zero shortfalls -- and zero shortfalls read as full support, making an unwritten claim the easiest kind to support. That is the vacuous-pass failure qc already guards against when a gate is evaluated with no measurements at all. Three entries were CONFIRMED while their own unverified field said the figures in demonstrates could not be checked at the cited source -- a paywall, a 403, a number that appears only in secondary coverage. That is the enum's written definition of PARTIAL, so they are PARTIAL now: ten confirmed, five partial. authority let a duplicate assignment suppress a violation. Plan.assigned returned the first matching row, so listing HUMAN before MODEL on batch release reported a clean plan -- a field an author could set to silence a report, in the module whose docstring promises none exists. Every row is judged now, and the duplication is its own finding rather than resolved, because picking a winner answers "who decides" on the author's behalf. cadence collapsed its own distinction one step after drawing it. cadence_for divided the face value of a RELEASED target and returned the STARTED interval under a RELEASED label, which is the exact substitution the endpoint refusal exists to prevent. It now asks required_admissions, which already returned None rather than assuming a survival fraction of 1.0, and refuses without one. An unverifiable date attached to a peak-day figure is removed; a date is the kind of detail that makes a number look verified. Adds the regression tests that would have caught all of them, including a behavioural test that plants a sustained-rate work and asserts the standing negative claim flips, so it is a computation rather than a permanently-false constant. 288 tests. --- README.md | 16 +++++--- autonomous_lab/authority.py | 45 +++++++++++++++++--- autonomous_lab/cadence.py | 26 ++++++++++-- autonomous_lab/cli.py | 18 +++++++- autonomous_lab/literature.py | 53 ++++++++++++++++++++++-- tests/test_authority.py | 33 +++++++++++++++ tests/test_cadence.py | 53 ++++++++++++++++++++++++ tests/test_literature.py | 80 +++++++++++++++++++++++++++++++++++- 8 files changed, 304 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 5b5c4dc..f20ecd3 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ autonomous-lab knowledge # encoded expert judgment and robo autonomous-lab evidence # what the published record shows, and what it does not autonomous-lab authority # which decisions a model may make, and which need a person -autonomous-lab cadence 500 --endpoint released --hours 16 # what that target actually costs +autonomous-lab cadence 500 --endpoint started --hours 16 # what that target actually costs ``` ## What it reports today @@ -331,7 +331,7 @@ is allowed to decide, and what does a stated throughput target actually cost?** ``` autonomous-lab evidence # what the record shows -- and what it does not autonomous-lab authority # which decisions a model may make -autonomous-lab cadence 500 --endpoint released --hours 16 +autonomous-lab cadence 500 --endpoint started --hours 16 ``` ### Evidence, scoped rather than cited @@ -342,8 +342,9 @@ production line. So every entry carries what it **does not establish**, and supp resolved by scope rather than by fame. Fifteen works, each independently verified against the live record before it was encoded -- -**thirteen confirmed, two partial, and eleven carrying figures that could not be -confirmed**, listed rather than rounded away. The verification changed the content: +**ten confirmed, five partial, and eleven carrying figures that could not be confirmed**, +listed rather than rounded away. An entry whose numbers were checked against a preprint +rather than the version of record is `PARTIAL`, however solid the work is. The verification changed the content: - The A-Lab paper was **corrected**. Forty-one successes became thirty-six after four were reclassified as inconclusive, and the "43 new materials" figure in news coverage matches @@ -383,7 +384,10 @@ A target is the easiest number in a lab to state and the easiest to state meanin `cadence` refuses one until it says what it counts, because plates *started*, *completed*, *QC-passed*, and *released* differ by exactly the failure and rerun rate -- a measurement, not an assumption. Resolving that silently would pick the loosest reading in the caller's -favour. +favour. It refuses twice: a downstream endpoint is a **return** rate, so reaching 500 +released means admitting more than 500, and without a measured survival fraction the module +will not convert. Dividing the face value would report the *started* interval under a +*released* label. Given an endpoint it is pure arithmetic on the caller's inputs, needing no measured durations: 500 plates a day is one every **172.8 s** round the clock, or **115.2 s** inside @@ -558,7 +562,7 @@ event receiver and silently steals the first one's callbacks. pip install -e '.[dev]' && pytest ``` -277 device-free tests. The ones that matter most try to make the layer lie: +288 device-free tests. The ones that matter most try to make the layer lie: - claim a step is automated when its command is undecoded; claim a decoded command is runnable while its siblings are not; claim a federated leg runs when no run card was ever diff --git a/autonomous_lab/authority.py b/autonomous_lab/authority.py index 1024c39..fd7e6ec 100644 --- a/autonomous_lab/authority.py +++ b/autonomous_lab/authority.py @@ -416,10 +416,19 @@ class Plan: assignments: Tuple[Assignment, ...] = () def assigned(self, decision_class: str) -> Optional[Assignment]: - for a in self.assignments: - if a.decision_class == decision_class: - return a - return None + """The single assignment for a class, or None when there is not exactly one. + + Returns None for a duplicated class rather than the first match. First-match is the + natural implementation and it is a hole in the invariant this module exists to enforce: + list the strong authority first and a weaker one after it, and the weaker row is never + read. `violations()` reports every row, so nothing hides behind an earlier one. + """ + found = self.all_assigned(decision_class) + return found[0] if len(found) == 1 else None + + def all_assigned(self, decision_class: str) -> Tuple[Assignment, ...]: + """Every assignment naming this class, in the order the plan lists them.""" + return tuple(a for a in self.assignments if a.decision_class == decision_class) @property def escalation_only(self) -> bool: @@ -443,6 +452,11 @@ class ViolationKind(str, Enum): WEAKENED = "weakened" # an authority below what the class requires UNASSIGNED = "unassigned" # the class exists and the plan is silent about it UNDECLARED = "undeclared" # the plan places a class this registry does not declare + # The plan assigns one class more than once. Reported in its own right rather than + # resolved, because every resolution rule picks a winner on the author's behalf and the + # ambiguity is the finding: a document that says a person releases the batch and also + # that a model does has not decided, and reading either row as the answer invents one. + DUPLICATED = "duplicated" @dataclass(frozen=True) @@ -490,7 +504,28 @@ class it does not assign at all. out: List[Violation] = [] for dc in registry: - assignment = plan.assigned(dc.name) + rows = plan.all_assigned(dc.name) + if len(rows) > 1: + weakest = min(rows, key=lambda a: _ORDER.index(a.authority)) + out.append( + Violation( + decision_class=dc.name, + kind=ViolationKind.DUPLICATED, + required=dc.required, + assigned=weakest.authority, + reason=( + f"'{dc.name}' is assigned {len(rows)} times (" + + ", ".join(a.authority.value for a in rows) + + f"). The plan has not decided who decides '{dc.decides}', and picking one row " + "would answer that on the author's behalf. The weakest is reported below on its " + "own merits" + ), + evidence_key=dc.evidence_key, + ) + ) + assignment = rows[0] if len(rows) == 1 else ( + min(rows, key=lambda a: _ORDER.index(a.authority)) if rows else None + ) if assignment is None: out.append( Violation( diff --git a/autonomous_lab/cadence.py b/autonomous_lab/cadence.py index 2afe32d..f82c02f 100644 --- a/autonomous_lab/cadence.py +++ b/autonomous_lab/cadence.py @@ -466,6 +466,7 @@ def cadence_for( operating_hours: float = 24.0, slowest_step_seconds: Optional[float] = None, end_to_end_seconds: Optional[float] = None, + yield_fraction: Optional[float] = None, ) -> Cadence: """Convert a stated target into a sustained admission interval. @@ -483,12 +484,28 @@ def cadence_for( ): if value is not None and value <= 0: raise ValueError(f"{name} of {value} is not a duration") - staffed = launch_interval(t.plates_per_day, operating_hours) + # An interval is an ADMISSION rate, and only STARTED is stated as one. A downstream target + # needs more admissions than its face value by exactly the rate at which work fails, so + # dividing the face value would return the STARTED interval under a RELEASED label -- the + # loosest reading, silently, which is the thing this module refuses one paragraph earlier. + # `required_admissions` already returns None rather than assuming a yield of 1.0; the bug + # was that this function did not ask it. + admissions = required_admissions(t, yield_fraction) + if admissions is None: + raise ValueError( + f"a target of {t.plates_per_day:g} plates {t.endpoint.value}/day is a RETURN rate, not " + "an admission rate. Reaching it means admitting more than that, by exactly the fraction " + "of work that fails between admission and this endpoint, and no benchmark in this " + "package measures it. Pass yield_fraction= to convert, or state the " + "target as plates STARTED. Dividing the face value would report the started interval " + "under this endpoint's name" + ) + staffed = launch_interval(admissions, operating_hours) return Cadence( target=t, operating_hours=operating_hours, interval_seconds=staffed, - round_the_clock_seconds=launch_interval(t.plates_per_day, 24.0), + round_the_clock_seconds=launch_interval(admissions, 24.0), slowest_step_seconds=slowest_step_seconds, end_to_end_seconds=end_to_end_seconds, ) @@ -757,8 +774,9 @@ def establishes_a_rate(self) -> bool: domain="inorganic solid-state powder synthesis, one prototype laboratory", ran_for="approximately 1.5 years of deployment", scale=( - "around 3,500 samples processed, with a stated peak of 149 samples on one day " - "(9 February 2024)" + "around 3,500 samples processed, with a stated peak of 149 samples on a single day. " + "The specific date is not restated here: it did not survive checking against the " + "paper, and a date is the kind of detail that makes a number look verified" ), demonstrated=Demonstrated.CAMPAIGN, actuates=True, diff --git a/autonomous_lab/cli.py b/autonomous_lab/cli.py index e716a75..c63d856 100644 --- a/autonomous_lab/cli.py +++ b/autonomous_lab/cli.py @@ -461,7 +461,19 @@ def _cadence(args) -> int: plates_per_day=args.plates_per_day, endpoint=Endpoint(args.endpoint.replace("-", "_")), ) - c = cadence_for(target, operating_hours=args.hours, slowest_step_seconds=args.slowest_step) + try: + c = cadence_for( + target, + operating_hours=args.hours, + slowest_step_seconds=args.slowest_step, + yield_fraction=args.yield_fraction, + ) + except ValueError as err: + print(f"cadence for {target.plates_per_day:g} plates {target.endpoint.value}/day\n") + print(" REFUSED") + print(f" {err}") + print("\n Re-run with --yield-fraction , or --endpoint started.") + return 1 print(f"cadence for {target.plates_per_day:g} plates {target.endpoint.value}/day\n") print( f" admission interval {c.interval_seconds:.1f} s " @@ -655,6 +667,10 @@ def common(sp): cd.add_argument("--intervention-rate", dest="intervention_rate", type=float, help="human interventions per plate") cd.add_argument("--minutes-per-intervention", dest="minutes_per_intervention", type=float, default=5.0) + cd.add_argument("--yield-fraction", dest="yield_fraction", type=float, + help="measured survival from admission to this endpoint; required for " + "completed/qc-passed/released, which are return rates rather than " + "admission rates") cd.set_defaults(func=_cadence) kn = sub.add_parser("knowledge", help="encoded expert judgment and robot benchmarks") diff --git a/autonomous_lab/literature.py b/autonomous_lab/literature.py index 1a09f17..82b4e42 100644 --- a/autonomous_lab/literature.py +++ b/autonomous_lab/literature.py @@ -142,6 +142,25 @@ class Scope: # nothing in this table is entitled to leave it empty. human_in_loop: str = "" + @property + def asserts_nothing_comparable(self) -> bool: + """True when a claim names no material, no duration, and no operating property. + + Such a claim is not modest, it is unwritten. Every comparison below is gated on the + claim asserting something, so an unwritten claim produces zero shortfalls -- and zero + shortfalls reads as full support. That is the same defect `qc` guards on the other side + of this package, where a gate evaluated against an empty measurement dict passes every + criterion vacuously. A missing assertion is not a satisfied one, and the ambition living + in `summary` prose is not a thing this module can check. + """ + return ( + not self.material.strip() + and self.continuous_hours is None + and not self.sustained + and not self.replicated + and not self.unattended + ) + @dataclass(frozen=True) class Evidence: @@ -408,7 +427,7 @@ def verified(self) -> bool: domain=Domain.CELL_ASSAY, kind=EvidenceKind.EMPIRICAL_DEMONSTRATION, year=2026, - confidence=Confidence.CONFIRMED, + confidence=Confidence.PARTIAL, shorthand="a 15-hour single-plate yeast run", demonstrates=( "A general-purpose robotic manipulator automating plate-based cell culture using vision " @@ -579,7 +598,7 @@ def verified(self) -> bool: domain=Domain.IN_VIVO, kind=EvidenceKind.EMPIRICAL_DEMONSTRATION, year=2023, - confidence=Confidence.CONFIRMED, + confidence=Confidence.PARTIAL, shorthand="a single-mouse home-cage phenotyping study", demonstrates=( "Automated cognitive testing in the home cage -- T-maze alternation, novel object " @@ -1074,7 +1093,7 @@ def verified(self) -> bool: domain=Domain.SAFETY, kind=EvidenceKind.BENCHMARK, year=2026, - confidence=Confidence.CONFIRMED, + confidence=Confidence.PARTIAL, shorthand="a text benchmark of 19 models on constructed lab-safety questions", demonstrates=( "Nineteen models evaluated -- 8 proprietary and 7 open-weight language models plus 4 " @@ -1234,6 +1253,34 @@ def _shortfalls(evidence: Evidence, scope: Scope) -> Tuple[str, ...]: "operating, so it cannot evidence an operating claim", ) + if scope.asserts_nothing_comparable: + return ( + "the claim names no material, duration, replication or attendance, so there is nothing " + f"to compare {short} against. Returning support here would mean an unwritten claim is " + "the easiest kind to support, which is backwards. Write the scope out and ask again", + ) + + # Material is compared before anything else that ran, because a duration gap on the wrong + # organism is a rounding error next to the organism. Comparison is exact-match and + # deliberately crude: "primary human hepatocytes" and "immortalized human cell line" are + # not machine-comparable, and a fuzzy match here would silently decide a transfer question + # that is the whole content of the claim. Unequal strings therefore refuse rather than + # guess, and say so. + if scope.material.strip(): + claimed = scope.material.strip().lower() + demonstrated = demo.material.strip().lower() + if not demonstrated: + out.append( + f"{short} states no material, so nothing in it compares against the claim's " + f"{scope.material}" + ) + elif claimed != demonstrated: + out.append( + f"the claim is about {scope.material}; {short} ran on {demo.material}. Whether a " + "result on one transfers to the other is exactly what a demonstration would have to " + "show, and nothing in this entry does" + ) + if scope.sustained and not demo.sustained: out.append( f"{short} is a single bounded campaign and does not reach sustained production" diff --git a/tests/test_authority.py b/tests/test_authority.py index 2211ec9..e7a3656 100644 --- a/tests/test_authority.py +++ b/tests/test_authority.py @@ -433,3 +433,36 @@ def test_nothing_in_a_plan_can_declare_itself_compliant(): # `complies` is computed from the registry and has no setter, so a plan cannot assert it. with pytest.raises(AttributeError): plan.complies = True + + +# -- no row may hide behind an earlier one --------------------------------------- + + +def test_a_stronger_assignment_listed_first_cannot_hide_a_weaker_one(): + """The module's stated invariant is that no field an author can set suppresses a + violation. A duplicate assignment was exactly such a field: first-match resolution meant + listing HUMAN before MODEL on batch release reported a clean plan. + """ + A = lambda auth: Assignment(decision_class="batch_release", authority=auth, who="x") + for order in ((A(Authority.HUMAN), A(Authority.MODEL)), (A(Authority.MODEL), A(Authority.HUMAN))): + found = [v for v in violations(Plan(name="p", assignments=order)) if v.decision_class == "batch_release"] + kinds = {v.kind for v in found} + assert ViolationKind.DUPLICATED in kinds, "the ambiguity itself is a finding" + assert ViolationKind.WEAKENED in kinds, "the weaker row must still be judged on its merits" + + +def test_a_duplicated_class_is_reported_rather_than_resolved(): + """Picking a winner would answer 'who decides' on the author's behalf. A plan naming a + class twice has not decided, and that is the thing to report.""" + A = lambda auth: Assignment(decision_class="batch_release", authority=auth, who="x") + plan = Plan(name="p", assignments=(A(Authority.HUMAN), A(Authority.MODEL))) + assert plan.assigned("batch_release") is None, "an ambiguous class resolves to nothing" + assert len(plan.all_assigned("batch_release")) == 2 + dup = next(v for v in violations(plan) if v.kind is ViolationKind.DUPLICATED) + assert "has not decided" in dup.reason + + +def test_a_single_correct_assignment_is_still_clean(): + """Guards the fix against overcorrecting into always-fail.""" + plan = Plan(name="p", assignments=(Assignment(decision_class="batch_release", authority=Authority.HUMAN, who="QA"),)) + assert not [v for v in violations(plan) if v.decision_class == "batch_release"] diff --git a/tests/test_cadence.py b/tests/test_cadence.py index 01fc60d..408ada0 100644 --- a/tests/test_cadence.py +++ b/tests/test_cadence.py @@ -488,3 +488,56 @@ def test_the_constraints_recompute_rather_than_being_stored(): def _forcing(cadence: Cadence, forcing: Forcing): return next(i for i in cadence.implied_constraints() if i.forcing is forcing) + + +# -- the negative claim must be behaviour, not a permanently-false constant -------- + + +def test_support_would_flip_if_a_sustained_rate_work_ever_appeared(): + """The module's headline is that nothing cited establishes a sustained rate. Proven only + against the current table, that passes just as well for an implementation hard-wired to + return nothing -- and it would keep passing after real evidence arrived. + + So inject one and check the behaviour flips. This is the test that makes the claim a + computation rather than a constant. + """ + import autonomous_lab.cadence as mod + + planted = mod.Demonstration( + **{ + **{f: getattr(mod.DEMONSTRATIONS[0], f) for f in mod.DEMONSTRATIONS[0].__dataclass_fields__}, + "key": "planted_sustained_rate", + "demonstrated": mod.Demonstrated.SUSTAINED_RATE, + } + ) + original = mod.DEMONSTRATIONS + try: + mod.DEMONSTRATIONS = original + (planted,) + assert mod.sustained_rate_demonstrations() == (planted,) + support = mod.demonstrated_support(mod.Target(500, mod.Endpoint.STARTED)) + assert support.supported, "a genuine sustained-rate work must flip the answer to yes" + assert planted in support.supporting() + finally: + mod.DEMONSTRATIONS = original + + # And the table is back, so the standing claim is negative again. + assert sustained_rate_demonstrations() == () + + +def test_a_downstream_target_without_a_measured_yield_is_refused_not_answered(): + """A released target is a RETURN rate. Dividing its face value returns the STARTED + interval under a RELEASED label -- the loosest reading, silently, which is the exact + substitution this module refuses one step earlier.""" + with pytest.raises(ValueError) as err: + cadence_for(Target(500, Endpoint.RELEASED)) + assert "admission rate" in str(err.value) + + # With a measured survival fraction it converts rather than refusing. + converted = cadence_for(Target(500, Endpoint.RELEASED), yield_fraction=0.8) + assert converted.interval_seconds == pytest.approx(138.24) + # Strictly shorter than the naive division, by exactly the failure rate. + assert converted.interval_seconds < launch_interval(500, 24) + + +def test_a_started_target_still_needs_no_yield(): + assert cadence_for(Target(500, Endpoint.STARTED)).interval_seconds == pytest.approx(172.8) diff --git a/tests/test_literature.py b/tests/test_literature.py index 79463cb..a839da2 100644 --- a/tests/test_literature.py +++ b/tests/test_literature.py @@ -128,7 +128,7 @@ def test_a_scope_that_sits_inside_a_demonstration_reports_as_supported(): """Guards against the cheapest possible bug: a checker that always answers no.""" modest = Scope( summary="a single ten-hour attended run on one plate of yeast", - material="yeast culture in a 96-well plate", + material=EVIDENCE_BY_KEY["roboculture"].scope.material, continuous_hours=10.0, ) support = support_for(Domain.CELL_ASSAY, modest) @@ -137,6 +137,59 @@ def test_a_scope_that_sits_inside_a_demonstration_reports_as_supported(): assert "roboculture" in covering +def test_a_claim_on_different_material_is_refused_even_when_everything_else_fits(): + """The load-bearing correctness test for scope. + + A ten-hour attended single-plate run is exactly what the cited work did -- on yeast. Ask + the same question about primary human cells and every other field still fits, so a + checker that ignores material returns a confident yes. That yes contradicts the entry's + own does_not_establish, which is the precise failure this module exists to prevent. + """ + hepatocytes = Scope( + summary="ten hours of primary human hepatocyte culture", + material="primary human hepatocytes", + continuous_hours=10.0, + ) + support = support_for(Domain.CELL_ASSAY, hepatocytes) + assert not support.supported + gap = _gap(support, "roboculture") + assert not gap.covers + # The refusal must name both materials, so a reader can judge the transfer themselves. + assert "primary human hepatocytes" in gap.reason + assert EVIDENCE_BY_KEY["roboculture"].scope.material in gap.reason + + +def test_material_comparison_refuses_rather_than_guessing_at_synonyms(): + """'yeast culture' and 'Saccharomyces cerevisiae culture' are the same thing to a reader + and not to a string comparison. Refusing is the correct behaviour: deciding that two + material descriptions are equivalent IS the transfer question, and a fuzzy match would + answer it silently in the direction of support.""" + near_synonym = Scope( + summary="ten hours of yeast", + material="yeast culture in a 96-well plate", + continuous_hours=10.0, + ) + assert not support_for(Domain.CELL_ASSAY, near_synonym).supported + + +def test_a_claim_that_asserts_nothing_comparable_is_refused_not_satisfied(): + """The other half of the same defect. Every comparison is gated on the claim asserting + something, so an unwritten claim yields zero shortfalls -- and zero shortfalls reads as + full support. That would make an unwritten claim the easiest kind to support, which is + backwards, and it is the same vacuous-pass failure `qc` guards against when a gate is + evaluated with no measurements at all. + """ + unwritten = Scope(summary="a continuous unattended sustained mammalian production line") + assert unwritten.asserts_nothing_comparable + support = support_for(Domain.CELL_ASSAY, unwritten) + assert not support.supported + assert "nothing to compare" in _gap(support, "roboculture").reason + + # Ambition stated in prose is not a scope. Naming one comparable field makes it checkable. + written = Scope(summary="same ambition, written down", sustained=True) + assert not written.asserts_nothing_comparable + + def test_the_same_demonstration_fails_the_same_claim_once_unattended_is_asserted(): """One field changes, and the citation that covered the claim stops covering it.""" attended = Scope(summary="ten attended hours", continuous_hours=10.0) @@ -435,3 +488,28 @@ def test_summary_adds_up(): assert s["domains_covered"] == len(Domain) assert s["unsupported_claims"] == len(UNSUPPORTED) assert s["entries_with_unverified_numbers"] >= s["partial"] + + +# -- confidence must match what verification actually achieved ------------------- + + +def test_an_entry_verified_only_against_a_preprint_is_not_confirmed(): + """`Confidence.CONFIRMED` means every number in `demonstrates` was checked at the cited + source. These three were not -- their own `unverified` says so, naming a paywall, a 403, + or a figure that appears only in secondary coverage. Marking them CONFIRMED would make + the distinction decorative, in the one module whose whole job is to keep it real. + """ + for key in ("roboculture", "labsafety", "smartkage"): + entry = EVIDENCE_BY_KEY[key] + assert entry.confidence is Confidence.PARTIAL, ( + f"{key} states figures it could not confirm at the source; it cannot be CONFIRMED" + ) + assert entry.unverified, f"{key} is PARTIAL and must say which figures are unconfirmed" + + +def test_every_partial_entry_names_what_it_could_not_confirm(): + """PARTIAL with an empty `unverified` is worse than CONFIRMED: it flags doubt and then + refuses to say where, which a reader cannot act on.""" + for e in EVIDENCE: + if e.confidence is Confidence.PARTIAL: + assert e.unverified, f"{e.key} is PARTIAL but names nothing it could not confirm" From 865444745f1f9e0a7b7965ab19ee11d6c5cc5edc Mon Sep 17 00:00:00 2001 From: di-omics Date: Tue, 28 Jul 2026 02:50:35 -0700 Subject: [PATCH 3/3] Use a def rather than a lambda in the duplicate-assignment tests ruff E731. The helper was a lambda for brevity in a test that constructs the same Assignment four times; a def reads the same and passes the gate the repo already runs. --- README.md | 2 +- tests/test_authority.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f20ecd3..accdba0 100644 --- a/README.md +++ b/README.md @@ -562,7 +562,7 @@ event receiver and silently steals the first one's callbacks. pip install -e '.[dev]' && pytest ``` -288 device-free tests. The ones that matter most try to make the layer lie: +292 device-free tests. The ones that matter most try to make the layer lie: - claim a step is automated when its command is undecoded; claim a decoded command is runnable while its siblings are not; claim a federated leg runs when no run card was ever diff --git a/tests/test_authority.py b/tests/test_authority.py index e7a3656..b35b676 100644 --- a/tests/test_authority.py +++ b/tests/test_authority.py @@ -443,7 +443,9 @@ def test_a_stronger_assignment_listed_first_cannot_hide_a_weaker_one(): violation. A duplicate assignment was exactly such a field: first-match resolution meant listing HUMAN before MODEL on batch release reported a clean plan. """ - A = lambda auth: Assignment(decision_class="batch_release", authority=auth, who="x") + def A(auth): + return Assignment(decision_class="batch_release", authority=auth, who="x") + for order in ((A(Authority.HUMAN), A(Authority.MODEL)), (A(Authority.MODEL), A(Authority.HUMAN))): found = [v for v in violations(Plan(name="p", assignments=order)) if v.decision_class == "batch_release"] kinds = {v.kind for v in found} @@ -454,7 +456,9 @@ def test_a_stronger_assignment_listed_first_cannot_hide_a_weaker_one(): def test_a_duplicated_class_is_reported_rather_than_resolved(): """Picking a winner would answer 'who decides' on the author's behalf. A plan naming a class twice has not decided, and that is the thing to report.""" - A = lambda auth: Assignment(decision_class="batch_release", authority=auth, who="x") + def A(auth): + return Assignment(decision_class="batch_release", authority=auth, who="x") + plan = Plan(name="p", assignments=(A(Authority.HUMAN), A(Authority.MODEL))) assert plan.assigned("batch_release") is None, "an ambiguous class resolves to nothing" assert len(plan.all_assigned("batch_release")) == 2