From 5b4e33fd448da91167198918634cd31b0a495d01 Mon Sep 17 00:00:00 2001 From: Ben Shaharizad Date: Sat, 19 Sep 2026 01:28:57 +0300 Subject: [PATCH 1/4] W5b-13 step 1 (engine-free): FieldSemantics on the public API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - jevmlx/api.py: frozen FieldSemantics dataclass (score_source / temperature / calibrator_id / prior_mode / constraint_changed / dependency_rescored) per the approved design note (GPT-REVIEW-2 §C9); FieldResult.semantics REQUIRED (keyword-only, no default) — constructing without it is a loud TypeError; _build_field_results passes telemetry.get('semantics') through (step 2 fills it in the engine stages). - README: FieldResult bullet documents semantics + the probability_status summary downgrade wording from the design note. - tests/test_field_semantics.py: dataclass contract (frozen, field names, required kw-only), export surface (single import path — the module-eviction identity trap), and the engine-filling pins marked xfail(strict=True) reason 'W5b-13 step 2' (verified: a simulated step-2 pass XPASSes and fails the suite, so the flip is loud). 672 passed, 2 xfailed; ruff clean. --- README.md | 10 ++ jevmlx/api.py | 49 +++++++++ tests/test_field_semantics.py | 180 ++++++++++++++++++++++++++++++++++ 3 files changed, 239 insertions(+) create mode 100644 tests/test_field_semantics.py diff --git a/README.md b/README.md index 5b3efc0..f4e5671 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,16 @@ Read the result. `decide(...)` returns a `Decision`: `.value` (a validated P(yes)), `score`, `model` (`"slots"`/`"labels"`), `calibrated`, `legal_mass` (probability mass in allowed continuations at the branch points — a leakage signal when low despite a confident decision). +- `semantics` (a frozen `FieldSemantics`): how THIS field's reported + probabilities were produced — which scoring path (`score_source`: + `batched` / `rescored_batch1` / `dependency` / `oracle`), the temperature + actually applied (`None` for count rows and calibrated multi selections, + whose log-odds cut ignores the caller temperature), the calibrator bundle + id when a fitted calibrator set the selection, the prior mode + (`off`/`neutral_v1`), and whether a constraint or a dependency wave + overrode the raw winner. The result-level `probability_status` summarizes + the distinct semantic groups and is not authoritative for any single + field. - Margins, one per field: `log_score_margin` / `probability_margin` (scalar, top1-top2 gap in log/probability units), `threshold_distance` (multi, how close the closest yes/no call sat to the cut). Multi fields carry diff --git a/jevmlx/api.py b/jevmlx/api.py index c078bc4..02c8c06 100644 --- a/jevmlx/api.py +++ b/jevmlx/api.py @@ -40,6 +40,7 @@ class Fraud(BaseModel): "NONE_OF_ABOVE_DESCRIPTION", "Decision", "FieldResult", + "FieldSemantics", "decide", "decide_many", "schema_from_model", @@ -80,6 +81,44 @@ def _choice_values(name: str, values: list) -> list[str]: return values +@dataclasses.dataclass(frozen=True) +class FieldSemantics: + """How ONE field's reported probabilities were produced (W5b-13, §C9). + + The per-field truth a global ``probability_status`` string cannot + carry: a result mixes temperature-scaled scalars, calibrated multi + options that ignore the caller temperature, count-row bucket scores, + prior-corrected fields, and dependency/oracle re-scores. Each field + records its own semantics; the result-level ``probability_status`` + becomes a summary of these records. + + Attributes: + score_source: Which scoring path produced the final evidence: + "batched" (batched pass), "rescored_batch1" (canonical batch=1 + rescore replaced it), "dependency" (second pass conditioned on + the parent), "oracle" (forced re-score under oracle_overrides). + temperature: The temperature actually applied to the reported + distribution. None for count rows (fixed T=1 bucket scores) + and for calibrated multi selections (the calibrated log-odds + ``a*(yes-no)+b`` cut ignores the caller temperature). + calibrator_id: Identity of the CalibrationBundle whose fitted + calibrator set the selection (multi only); None = uncalibrated. + prior_mode: "off" or "neutral_v1" — whether (and how) the scores + were prior-corrected against the neutral-context pass. + constraint_changed: A reconciler (count / set constraints / case + MAP) overrode the raw winner. + dependency_rescored: Re-scored conditioned on the parent value in + a dependency wave. + """ + + score_source: str + temperature: float | None + calibrator_id: str | None + prior_mode: str + constraint_changed: bool + dependency_rescored: bool + + @dataclasses.dataclass(frozen=True) class FieldResult: """Provenance for one decided field. @@ -119,6 +158,11 @@ class FieldResult: gate (calibrated abstention is a later milestone). The only values are None, "none_of_above" and "abstain"; a withheld decision is exactly ``reason == "abstain"``. + semantics: HOW this field's reported probabilities were produced — + see :class:`FieldSemantics`. Required, never None: the engine + fills it in step 2 of W5b-13; until then constructing a + FieldResult without it fails loudly rather than implying a + default reading. """ value: object @@ -131,6 +175,7 @@ class FieldResult: model: str alternatives: tuple[tuple[str, float], ...] reason: str | None = None + semantics: FieldSemantics = dataclasses.field(kw_only=True) # type: ignore[assignment] @dataclasses.dataclass @@ -368,6 +413,10 @@ def _build_field_results( model=confidence_model, alternatives=alternatives, reason=reason, + # W5b-13 step 2 fills this from the engine's per-stage records; + # until then the API surface REQUIRES the caller to have one — + # tests construct it explicitly (no silent default reading). + semantics=telemetry.get("semantics"), ) return fields diff --git a/tests/test_field_semantics.py b/tests/test_field_semantics.py new file mode 100644 index 0000000..d139939 --- /dev/null +++ b/tests/test_field_semantics.py @@ -0,0 +1,180 @@ +"""W5b-13 step 1: FieldSemantics contract tests (design-note C9). + +The dataclass lives on the public API NOW; the ENGINE fills it in step 2 +(set in the #50 stages: score_scalar_field / score_multi_field / +reconcile_case_constraints / run_dependency_waves / finalize_public_result). +Until step 2 lands, decide() cannot supply semantics — the end-to-end +pinning test is xfail(strict=True) so it fails loudly today and flips the +suite green exactly when step 2 removes the mark. +""" + +from __future__ import annotations + +import dataclasses +from typing import Literal + +import pytest +from pydantic import BaseModel, Field + +import jevmlx +from jevmlx.api import FieldResult, FieldSemantics + +# --- the dataclass contract ---------------------------------------------------- + + +def test_field_semantics_frozen_and_field_names(): + """Frozen dataclass; the six C9 field names in the design order.""" + s = FieldSemantics( + score_source="batched", + temperature=1.0, + calibrator_id=None, + prior_mode="off", + constraint_changed=False, + dependency_rescored=False, + ) + assert dataclasses.is_dataclass(s) + with pytest.raises(dataclasses.FrozenInstanceError): + s.temperature = 0.7 # type: ignore[misc] + assert [f.name for f in dataclasses.fields(s)] == [ + "score_source", + "temperature", + "calibrator_id", + "prior_mode", + "constraint_changed", + "dependency_rescored", + ] + + +def test_field_result_requires_semantics(): + """FieldResult.semantics is REQUIRED (keyword-only, no default): a + construction without it is a TypeError, never a silent default.""" + kwargs = dict( + value="A", + score=0.0, + log_score_margin=None, + probability_margin=None, + threshold_distance=None, + probability=0.9, + calibrated=False, + model="slots", + alternatives=(), + ) + with pytest.raises(TypeError, match="semantics"): + FieldResult(**kwargs) + s = FieldSemantics("batched", 1.0, None, "off", False, False) + fr = FieldResult(**kwargs, semantics=s) + assert fr.semantics is s + + +def test_field_semantics_exported(): + """Public surface: importable from jevmlx.api, listed in __all__. + + Both bindings come from ONE import path inside the test: the + test_check_results module eviction can leave this file's collection-time + binding on a stale module instance, and cross-instance identity would + fail for the wrong reason (see test_bench_machine_tag_patches_live_module).""" + import jevmlx.api as api + from jevmlx.api import FieldSemantics as FS # noqa: N806 — same-path binding + + assert api.FieldSemantics is FS + assert "FieldSemantics" in api.__all__ + + +# --- engine-filling pinning test (step 2) -------------------------------------- + + +def _run_pinned_decide(monkeypatch): + """decide() over the shared fake engine with one scalar field: the + minimal path whose FieldResult must carry a complete semantics record + once the engine sets it (W5b-13 step 2).""" + import jevmlx.api as api + from tests.conftest import make_engine_result, make_field_telemetry + + monkeypatch.setattr( + api, + "run_parallel_generation", + lambda *a, **k: make_engine_result( + fields={ + "risk_tier": make_field_telemetry( + value="HIGH", + choices=["HIGH", "LOW", "CRITICAL"], + ) + }, + ), + ) + monkeypatch.setattr(api, "load_engine", lambda model_id: ("engine", "tokenizer")) + + class Ticket(BaseModel): + risk_tier: Literal["HIGH", "LOW", "CRITICAL"] = Field(description="Risk tier") + + return jevmlx.decide(Ticket, "ctx", model="fake/model") + + +@pytest.mark.xfail( + strict=True, + reason="W5b-13 step 2: the engine does not set FieldResult.semantics yet " + "(lands after the '#51 merged' ping; #50 stages set it per the design table)", +) +def test_engine_fills_semantics_per_field(monkeypatch): + """PIN (step 2): the engine's decided fields carry per-field semantics — + scalar path: batched evidence, the caller temperature, no calibrator, + prior off, no constraint change, no dependency rescore.""" + d = _run_pinned_decide(monkeypatch) + fr = d.fields["risk_tier"] + assert isinstance(fr.semantics, FieldSemantics) + assert fr.semantics == FieldSemantics( + score_source="batched", + temperature=1.0, + calibrator_id=None, + prior_mode="off", + constraint_changed=False, + dependency_rescored=False, + ) + + +@pytest.mark.xfail( + strict=True, + reason="W5b-13 step 2: probability_status stays the old global string until " + "the stages set semantics and finalize_public_result builds the summary", +) +def test_probability_status_summarizes_semantics_groups(monkeypatch): + """PIN (step 2): the result-level probability_status is a SUMMARY over the + distinct (score_source, temperature, calibrator_id, prior_mode) groups — + it names the count of fields per group and never a per-field claim.""" + d = _run_pinned_decide(monkeypatch) + status = d.fields["risk_tier"].semantics # placeholder to force attr use + result_status = _run_result_status(monkeypatch) + assert isinstance(status, FieldSemantics) + # One distinct group (scalar, T=1, uncalibrated, prior off) -> the + # summary must mention that group, with no per-field value claims. + assert "1 field" in result_status + assert "batched" in result_status + assert "T=1" in result_status + + +def _run_result_status(monkeypatch) -> str: + """The raw engine result's probability_status through the fake path.""" + import jevmlx.api as api + from tests.conftest import make_engine_result, make_field_telemetry + + res = make_engine_result( + fields={ + "risk_tier": make_field_telemetry( + value="HIGH", + choices=["HIGH", "LOW", "CRITICAL"], + ) + }, + ) + monkeypatch.setattr(api, "run_parallel_generation", lambda *a, **k: res) + monkeypatch.setattr(api, "load_engine", lambda model_id: ("engine", "tokenizer")) + + from typing import Literal as L + + from pydantic import BaseModel + + class Ticket(BaseModel): + risk_tier: L["HIGH", "LOW", "CRITICAL"] = Field(description="Risk tier") + + d = jevmlx.decide(Ticket, "ctx", model="fake/model") + assert d.fields["risk_tier"].semantics is not None # step-2 gate + return res["probability_status"] From 2205f6ed7e9034b14738f0b017911814bccd80b1 Mon Sep 17 00:00:00 2001 From: Ben Shaharizad Date: Sat, 19 Sep 2026 01:55:43 +0300 Subject: [PATCH 2/4] W5b-13 review fixes (F1-F3) on the draft PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - F1: FieldResult is now @dataclass(frozen=True, kw_only=True) — semantics loses the kw_only-field + type: ignore hack; 'reason' loses its default (all fields kw_only). Callers updated; the one production constructor (_build_field_results) passes every field by name already. - F2: placeholder summary test deleted; test_probability_status_is_a_summary rewritten cleanly against the step-2 behavior (helper returns the engine status, no duplicated fixture, no placeholder line). - F3: README semantics bullet removed — ships with step 2. - FieldResult.semantics docstring now states the draft truth: the PR does not leave draft until every decided field carries a real record. - Kept: frozen FieldSemantics, the two contract tests, xfail(strict=True) pins (strictness re-verified: simulated step-2 pass XPASSes and fails). 672 passed, 2 xfailed; ruff clean. --- README.md | 10 --------- jevmlx/api.py | 22 +++++++++++--------- tests/test_field_semantics.py | 38 +++++++++++++++++------------------ 3 files changed, 30 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index f4e5671..5b3efc0 100644 --- a/README.md +++ b/README.md @@ -72,16 +72,6 @@ Read the result. `decide(...)` returns a `Decision`: `.value` (a validated P(yes)), `score`, `model` (`"slots"`/`"labels"`), `calibrated`, `legal_mass` (probability mass in allowed continuations at the branch points — a leakage signal when low despite a confident decision). -- `semantics` (a frozen `FieldSemantics`): how THIS field's reported - probabilities were produced — which scoring path (`score_source`: - `batched` / `rescored_batch1` / `dependency` / `oracle`), the temperature - actually applied (`None` for count rows and calibrated multi selections, - whose log-odds cut ignores the caller temperature), the calibrator bundle - id when a fitted calibrator set the selection, the prior mode - (`off`/`neutral_v1`), and whether a constraint or a dependency wave - overrode the raw winner. The result-level `probability_status` summarizes - the distinct semantic groups and is not authoritative for any single - field. - Margins, one per field: `log_score_margin` / `probability_margin` (scalar, top1-top2 gap in log/probability units), `threshold_distance` (multi, how close the closest yes/no call sat to the cut). Multi fields carry diff --git a/jevmlx/api.py b/jevmlx/api.py index 02c8c06..8075403 100644 --- a/jevmlx/api.py +++ b/jevmlx/api.py @@ -119,7 +119,7 @@ class FieldSemantics: dependency_rescored: bool -@dataclasses.dataclass(frozen=True) +@dataclasses.dataclass(frozen=True, kw_only=True) class FieldResult: """Provenance for one decided field. @@ -159,10 +159,11 @@ class FieldResult: only values are None, "none_of_above" and "abstain"; a withheld decision is exactly ``reason == "abstain"``. semantics: HOW this field's reported probabilities were produced — - see :class:`FieldSemantics`. Required, never None: the engine - fills it in step 2 of W5b-13; until then constructing a - FieldResult without it fails loudly rather than implying a - default reading. + see :class:`FieldSemantics`. Required: the engine fills it in + W5b-13 step 2 (this draft's decide() passes + telemetry.get("semantics") through, which is None until then; + the PR does not leave draft until every decided field carries a + real record). """ value: object @@ -174,8 +175,8 @@ class FieldResult: calibrated: bool model: str alternatives: tuple[tuple[str, float], ...] - reason: str | None = None - semantics: FieldSemantics = dataclasses.field(kw_only=True) # type: ignore[assignment] + reason: str | None + semantics: FieldSemantics @dataclasses.dataclass @@ -413,9 +414,10 @@ def _build_field_results( model=confidence_model, alternatives=alternatives, reason=reason, - # W5b-13 step 2 fills this from the engine's per-stage records; - # until then the API surface REQUIRES the caller to have one — - # tests construct it explicitly (no silent default reading). + # W5b-13 step 2 fills this from the engine's per-stage records + # (telemetry.get('semantics') is None until then — acceptable + # ONLY inside the draft PR; decide() does not ship to ready + # state until the engine sets it). semantics=telemetry.get("semantics"), ) return fields diff --git a/tests/test_field_semantics.py b/tests/test_field_semantics.py index d139939..1b891b8 100644 --- a/tests/test_field_semantics.py +++ b/tests/test_field_semantics.py @@ -46,8 +46,8 @@ def test_field_semantics_frozen_and_field_names(): def test_field_result_requires_semantics(): - """FieldResult.semantics is REQUIRED (keyword-only, no default): a - construction without it is a TypeError, never a silent default.""" + """FieldResult is kw_only-frozen and semantics is REQUIRED (no default): + a construction without it is a TypeError, never a silent default.""" kwargs = dict( value="A", score=0.0, @@ -58,6 +58,7 @@ def test_field_result_requires_semantics(): calibrated=False, model="slots", alternatives=(), + reason=None, ) with pytest.raises(TypeError, match="semantics"): FieldResult(**kwargs) @@ -137,22 +138,23 @@ def test_engine_fills_semantics_per_field(monkeypatch): reason="W5b-13 step 2: probability_status stays the old global string until " "the stages set semantics and finalize_public_result builds the summary", ) -def test_probability_status_summarizes_semantics_groups(monkeypatch): - """PIN (step 2): the result-level probability_status is a SUMMARY over the - distinct (score_source, temperature, calibrator_id, prior_mode) groups — - it names the count of fields per group and never a per-field claim.""" +def test_probability_status_is_a_summary(monkeypatch): + """PIN (step 2, written cleanly against the then-existing behavior): the + result-level probability_status is a SUMMARY over the distinct + (score_source, temperature, calibrator_id, prior_mode) groups — field + counts per group, no per-field probability claims. Replaces the + placeholder draft deleted in review (F2).""" d = _run_pinned_decide(monkeypatch) - status = d.fields["risk_tier"].semantics # placeholder to force attr use - result_status = _run_result_status(monkeypatch) - assert isinstance(status, FieldSemantics) - # One distinct group (scalar, T=1, uncalibrated, prior off) -> the - # summary must mention that group, with no per-field value claims. - assert "1 field" in result_status - assert "batched" in result_status - assert "T=1" in result_status + assert isinstance(d.fields["risk_tier"].semantics, FieldSemantics) + status = _engine_probability_status(monkeypatch) + # One distinct group (scalar, T=1, uncalibrated, prior off): the summary + # mentions that group with its field count and no per-field values. + assert "1 field" in status + assert "batched" in status + assert "T=1" in status -def _run_result_status(monkeypatch) -> str: +def _engine_probability_status(monkeypatch) -> str: """The raw engine result's probability_status through the fake path.""" import jevmlx.api as api from tests.conftest import make_engine_result, make_field_telemetry @@ -168,12 +170,8 @@ def _run_result_status(monkeypatch) -> str: monkeypatch.setattr(api, "run_parallel_generation", lambda *a, **k: res) monkeypatch.setattr(api, "load_engine", lambda model_id: ("engine", "tokenizer")) - from typing import Literal as L - - from pydantic import BaseModel - class Ticket(BaseModel): - risk_tier: L["HIGH", "LOW", "CRITICAL"] = Field(description="Risk tier") + risk_tier: Literal["HIGH", "LOW", "CRITICAL"] = Field(description="Risk tier") d = jevmlx.decide(Ticket, "ctx", model="fake/model") assert d.fields["risk_tier"].semantics is not None # step-2 gate From 6edbcffe42ba575d854315eaa8238d40b3df230c Mon Sep 17 00:00:00 2001 From: Ben Shaharizad Date: Sat, 19 Sep 2026 11:51:42 +0300 Subject: [PATCH 3/4] W5b-13 step 2: engine fills FieldSemantics in the stages; probability_status becomes the group summary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - engine: _field_semantics() is the ONE constructor for the semantics record (dict — the api import graph stays one-way; api.FieldSemantics coerces at the public boundary). - score_scalar_field: score_source from the finalizer's evidence_source (batch -> batched, batch1 -> rescored_batch1); temperature = the fitted scalar T when the bundle applied it, the caller T otherwise; prior_mode from prior_corrected. _cardinality_one_outcome: schema-determined record (temperature None, nothing corrected). - score_multi_field: score_source mirrors the scalar vocabulary (rescored_batch1 when any Y/N pair was band-rescored); temperature None when the calibrated log-odds cut set the selection (it ignores T); calibrator_id from the bundle; constraint_changed from solve_multi_set's reconciled_by. Count row: temperature None (fixed T=1 buckets), its own constraint_changed. - reconcile_case_constraints + the post-dependency MAP: flips set constraint_changed=True on exactly the changed fields. - _selective_second_pass: a dependency re-decide REPLACES the field's semantics (score_source dependency/oracle, dependency_rescored True, prior constraint_changed carried over). - finalize_public_result: probability_status is the group-count SUMMARY — one clause per distinct (score_source, temperature, calibrator_id, prior_mode) group with its field count; the old global-only statement is deleted (kept only as the no-records fallback for fake/baseline paths). - api._build_field_results: coerces telemetry['semantics'] into the frozen FieldSemantics; a missing record is a ValueError naming the field — 'required' is now true at every decide() return. - tests: conftest make_field_semantics factory (default record on every fake telemetry shape); test_field_semantics.py rewritten one-pin-per- behavior (decide coercion, loud missing-record failure, group summary + prior/calibrator clauses via finalize_public_result, engine shapes, constraint flips, calibrated split); old global-string pins updated; xfail marks dropped. - README: semantics bullet + probability_status summary wording. 746 passed -m 'not slow', 2 slow pins green, ruff clean. --- README.md | 11 + jevmlx/api.py | 18 +- jevmlx/engine.py | 179 +++++++++++++++-- tests/conftest.py | 22 ++ tests/test_engine.py | 17 +- tests/test_engine_fake.py | 18 +- tests/test_field_semantics.py | 368 ++++++++++++++++++++++++++-------- 7 files changed, 517 insertions(+), 116 deletions(-) diff --git a/README.md b/README.md index 5b3efc0..786fe8f 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,17 @@ Read the result. `decide(...)` returns a `Decision`: `.value` (a validated P(yes)), `score`, `model` (`"slots"`/`"labels"`), `calibrated`, `legal_mass` (probability mass in allowed continuations at the branch points — a leakage signal when low despite a confident decision). +- `semantics` (a frozen `FieldSemantics`): how THIS field's reported + probabilities were produced — which scoring path (`score_source`: + `batched` / `rescored_batch1` / `dependency` / `oracle`), the temperature + actually applied (`None` for count rows and calibrated multi selections, + whose log-odds cut ignores the caller temperature), the calibrator bundle + id when a fitted calibrator set the selection, the prior mode + (`off`/`neutral_v1`), and whether a constraint or a dependency wave + overrode the raw winner. The result-level `probability_status` summarizes + the distinct semantic groups (one clause per + `(score_source, temperature, calibrator_id, prior_mode)` group with its + field count) and is not authoritative for any single field. - Margins, one per field: `log_score_margin` / `probability_margin` (scalar, top1-top2 gap in log/probability units), `threshold_distance` (multi, how close the closest yes/no call sat to the cut). Multi fields carry diff --git a/jevmlx/api.py b/jevmlx/api.py index 8075403..fbcd3bd 100644 --- a/jevmlx/api.py +++ b/jevmlx/api.py @@ -403,6 +403,18 @@ def _build_field_results( # Provenance (the bundle identity) belongs to the telemetry, not to # confidence_model: model stays the clean decision-model name. calibrated_flag = telemetry.get("calibrated") is not None + # W5b-13 step 2: the engine's stages set the semantics record; the + # public API coerces it into the frozen FieldSemantics. A telemetry + # entry without one is a contract violation — fail loudly, never + # ship a None semantics (the docstring's 'required' is now TRUE at + # every decide() return). + sem_dict = telemetry.get("semantics") + if not isinstance(sem_dict, dict): + raise ValueError( + f"field telemetry for {name!r} carries no semantics record " + "(results-contract violation; engine stages must set " + "field_telemetry[fname]['semantics'])" + ) fields[name] = FieldResult( value=telemetry["value"], score=score, @@ -414,11 +426,7 @@ def _build_field_results( model=confidence_model, alternatives=alternatives, reason=reason, - # W5b-13 step 2 fills this from the engine's per-stage records - # (telemetry.get('semantics') is None until then — acceptable - # ONLY inside the draft PR; decide() does not ship to ready - # state until the engine sets it). - semantics=telemetry.get("semantics"), + semantics=FieldSemantics(**sem_dict), ) return fields diff --git a/jevmlx/engine.py b/jevmlx/engine.py index ed7d02e..88a13b2 100644 --- a/jevmlx/engine.py +++ b/jevmlx/engine.py @@ -854,6 +854,37 @@ class Candidate: score: float +def _field_semantics( + *, + score_source: str, + temperature: float | None, + calib: "CalibrationBundle | None", + calibrated_applied: bool, + prior_corrected: bool, + constraint_changed: bool = False, + dependency_rescored: bool = False, +) -> dict[str, Any]: + """The per-field semantics record (W5b-13, GPT-REVIEW-2 §C9). + + ONE constructor for the ``semantics`` telemetry dict every scoring + stage attaches to its field: which path produced the evidence, the + temperature actually applied, the calibrator bundle id when a fitted + calibrator set the selection, the prior mode, and whether a + reconciler or a dependency wave overrode the raw winner. The public + API coerces this dict into the frozen ``api.FieldSemantics``; + emitting the plain dict here keeps the engine free of an api import + (api imports engine, not the reverse). + """ + return { + "score_source": score_source, + "temperature": temperature, + "calibrator_id": calib.identity() if (calib is not None and calibrated_applied) else None, + "prior_mode": "neutral_v1" if prior_corrected else "off", + "constraint_changed": constraint_changed, + "dependency_rescored": dependency_rescored, + } + + def finalize_scalar_evidence( evidence: ScalarEvidence, *, @@ -1921,6 +1952,22 @@ def mass_at_node(node: dict, _l=branch_mass, _i=branch_index) -> float: field_telemetry[fname]["legal_mass_logs"] = dict(decision.legal_mass_logs) field_telemetry[fname]["second_pass"] = True field_telemetry[fname]["evidence_source"] = decision.evidence_source + # W5b-13: a dependency re-decide REPLACES the field's semantics — + # the reported probabilities now come from the conditioned pass. + # A MAP reconciliation that happened EARLIER stays recorded + # (constraint_changed carries over); the oracle path records + # without touching the main predictions, so its semantics live + # only on the fields it re-decided. + _prev_sem = field_telemetry[fname].get("semantics") or {} + field_telemetry[fname]["semantics"] = _field_semantics( + score_source="oracle" if is_oracle else "dependency", + temperature=temperature, + calib=None, + calibrated_applied=False, + prior_corrected=decision.prior_corrected, + constraint_changed=_prev_sem.get("constraint_changed", False), + dependency_rescored=not is_oracle, + ) rerun_fields.append(fname) assignments[fname] = val affected.add(fname) @@ -1945,6 +1992,9 @@ def mass_at_node(node: dict, _l=branch_mass, _i=branch_index) -> float: if fname in parsed_json and parsed_json[fname]["value"] != val: parsed_json[fname]["value"] = val field_telemetry[fname]["value"] = val + # W5b-13: the post-dependency MAP changed this field again. + if "semantics" in field_telemetry[fname]: + field_telemetry[fname]["semantics"]["constraint_changed"] = True final_assignment = {fname: pj["value"] for fname, pj in parsed_json.items()} if not compiled_constraints.satisfied(final_assignment): raise InternalConstraintViolationError( @@ -2641,6 +2691,16 @@ def _cardinality_one_outcome( "top_choices": [{"choice": key, "probability": 1.0}], "rows": 0, "legal_mass": 1.0, + # W5b-13: cardinality-1 fields are schema-determined — no model + # scoring, no temperature applied, nothing corrected. + "semantics": { + "score_source": "batched", + "temperature": None, + "calibrator_id": None, + "prior_mode": "off", + "constraint_changed": False, + "dependency_rescored": False, + }, }, ) @@ -2731,6 +2791,20 @@ def score_scalar_field( {"temperature": calib.temperature} if calib is not None and calib.has_scalar else None ) calibration_id = calib.identity() if scalar_calibrated is not None else None + # W5b-13: the per-field semantics record. Evidence source from the ONE + # finalizer (batch -> batched, batch1 -> rescored_batch1); the applied + # temperature is the fitted scalar T when the bundle supplied it, the + # caller temperature otherwise. + applied_temperature = ( + calib.temperature if calib is not None and calib.has_scalar else temperature + ) + semantics = _field_semantics( + score_source="rescored_batch1" if decision.rescored else "batched", + temperature=applied_temperature, + calib=calib, + calibrated_applied=scalar_calibrated is not None, + prior_corrected=decision.prior_corrected, + ) telemetry = { "value": val, "type": fdef.field_type, @@ -2754,6 +2828,8 @@ def score_scalar_field( # FULL vocabulary (raw, pre-prior-correction). "legal_mass": decision.legal_mass, "legal_mass_logs": dict(decision.legal_mass_logs), + # W5b-13: how this field's reported probabilities were produced. + "semantics": semantics, } if decision.prior_corrected: telemetry["prior_log_scores"] = dict(decision.prior_log_scores) @@ -3218,6 +3294,36 @@ def score_multi_field( prior_pairs, selected, ) + # W5b-13: multi semantics. score_source mirrors the scalar vocabulary — + # 'rescored_batch1' when any option's Y/N pair was band-rescored. The + # caller temperature was applied to the P(yes) softmax, BUT the + # calibrated selection (a*(yes-no)+b, calib.has_multi) ignores it — the + # record's temperature is None in that case so nobody reads a T that + # did not set the decision. + multi_temperature = None if (calib is not None and calib.has_multi) else temperature + telemetry["semantics"] = _field_semantics( + score_source="rescored_batch1" if multi_rescored else "batched", + temperature=multi_temperature, + calib=calib, + calibrated_applied=calib is not None and calib.has_multi, + prior_corrected=prior_entry is not None, + # solve_multi_set's reconciled_by: 'per_option' = the raw threshold + # proposal stood; 'count' or a setcons rule name = a reconciler + # overrode it (W5b-13 constraint_changed). + constraint_changed=solved_telemetry.get("reconciled_by") != "per_option", + ) + # The count row rides the parent multi's prior mode and score path; its + # bucket scores are fixed T=1 softmaxes (never temperature-scaled) and + # its selection is the trusted-count rule (a reconciler by nature). + if count_telemetry is not None: + count_telemetry["semantics"] = _field_semantics( + score_source="rescored_batch1" if multi_rescored else "batched", + temperature=None, + calib=None, + calibrated_applied=False, + prior_corrected=prior_entry is not None, + constraint_changed=count_telemetry.get("dropped_reason") is None, + ) return FieldOutcome( fname, {"value": selected, "prob": None}, @@ -3274,6 +3380,10 @@ def reconcile_case_constraints( field_telemetry[fname]["probability"] = math.exp( field_log_scores[fname][str(val)] ) + # W5b-13: the MAP overrode the raw winner — the field's + # semantics record says so. + if "semantics" in field_telemetry[fname]: + field_telemetry[fname]["semantics"]["constraint_changed"] = True return AssembledState( parsed_json=parsed_json, field_telemetry=field_telemetry, @@ -3362,22 +3472,61 @@ def finalize_public_result( # shared prior_ms is passed in and exposed on every result (finding 26). flat = ledger.derived_flat(prior_ms=prior_ms) total_elapsed_ms = flat["elapsed_ms"] - # Bug 12: probability_status must tell the truth about the temperature. - # At T=1 the reported distribution is the constrained-path probability; - # at any other temperature it is a post-hoc temperature-scaled - # distribution and the temperature is part of the statement. - if temperature == 1.0: - probability_status = ( - "constrained-path probability at T=1; uncalibrated as decision confidence" - ) + # Bug 12 / W5b-13: probability_status is a SUMMARY over the per-field + # semantics records — one clause per distinct + # (score_source, temperature, calibrator_id, prior_mode) group with its + # field count. The per-field truth lives in field_telemetry[..]['semantics'] + # (api.FieldSemantics); this string is not authoritative for any single + # field. The old global-only statement is gone: a result mixing + # temperature-scaled scalars, calibrated multi options and dependency + # re-scores cannot be described by one sentence. + semantic_groups: dict[tuple, list[str]] = {} + for fname, ft in state.field_telemetry.items(): + sem = ft.get("semantics") + if not isinstance(sem, dict): + continue + key = (sem["score_source"], sem["temperature"], sem["calibrator_id"], sem["prior_mode"]) + semantic_groups.setdefault(key, []).append(fname) + clauses: list[str] = [] + for (source, temp, cal_id, pmode), names in sorted( + semantic_groups.items(), key=lambda kv: (-len(kv[1]), kv[0][0]) + ): + n = len(names) + noun = "field" if n == 1 else "fields" + parts = [f"{n} {noun}: {source}"] + if temp is None: + parts.append("temperature not applied (fixed rule selection)") + else: + t_clause = ( + "constrained-path probability" + if temp == 1.0 + else f"post-hoc temperature-scaled (T={temp})" + ) + parts.append(t_clause) + if cal_id is not None: + parts.append(f"calibrated (bundle {cal_id})") + else: + parts.append("uncalibrated as decision confidence") + if pmode == "neutral_v1": + parts.append("prior-corrected against the neutral-context pass") + clauses.append("; ".join(parts)) + if clauses: + probability_status = " per distinct semantics group | ".join(clauses) else: - probability_status = ( - f"post-hoc temperature-scaled constrained distribution " - f"(temperature={temperature}); ranking-invariant, not a T=1 probability; " - f"uncalibrated as decision confidence" - ) - if prior_correction: - probability_status += "; prior-corrected against the neutral-context pass" + # No semantics records (fake/baseline paths): keep the classic + # temperature-honest statement rather than an empty string. + if temperature == 1.0: + probability_status = ( + "constrained-path probability at T=1; uncalibrated as decision confidence" + ) + else: + probability_status = ( + f"post-hoc temperature-scaled constrained distribution " + f"(temperature={temperature}); ranking-invariant, not a T=1 probability; " + f"uncalibrated as decision confidence" + ) + if prior_correction: + probability_status += "; prior-corrected against the neutral-context pass" # Bug 9 / W5b-14: the timing split is honest about the whole request # wall time and every key is a ledger derivation (prior_ms = the prior diff --git a/tests/conftest.py b/tests/conftest.py index 48d40f9..1529a02 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -195,6 +195,27 @@ def _base_field_telemetry() -> dict: } +def make_field_semantics( + score_source: str = "batched", + temperature: float | None = 1.0, + calibrator_id: str | None = None, + prior_mode: str = "off", + constraint_changed: bool = False, + dependency_rescored: bool = False, +) -> dict: + """W5b-13: the engine's per-field semantics record (the dict the engine + stages emit; api.FieldSemantics(**d) coerces it at the public boundary). + Same defaults the engine produces on a plain uncorrected fake path.""" + return { + "score_source": score_source, + "temperature": temperature, + "calibrator_id": calibrator_id, + "prior_mode": prior_mode, + "constraint_changed": constraint_changed, + "dependency_rescored": dependency_rescored, + } + + def make_field_telemetry( value: object = "A", type_: str = "enum", @@ -291,6 +312,7 @@ def make_field_telemetry( "legal_mass": 1.0, "legal_mass_logs": {c: 0.0 for c in opts}, } + entry.setdefault("semantics", make_field_semantics()) entry.update(overrides) return entry diff --git a/tests/test_engine.py b/tests/test_engine.py index af50b7d..9c318ae 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -412,18 +412,21 @@ def test_timing_split_on_real_model(engine): @pytest.mark.slow def test_probability_status_temperature_on_real_model(engine): - """Bug 12 on a real model: T=1 keeps the classic status; T!=1 states the - post-hoc scaling and the temperature value.""" + """Bug 12 + W5b-13 on a real model: the status is the per-group summary — + the T clause carries the temperature (T=1 = constrained-path, T!=1 = + post-hoc scaling with the value).""" schema = StructuredSchema( {"tier": {"type": "enum", "description": "d", "choices": ["LOW", "HIGH"]}} ) at_one = run_parallel_generation(engine, "ctx", schema, temperature=1.0) - assert at_one["probability_status"] == ( - "constrained-path probability at T=1; uncalibrated as decision confidence" - ) + status_one = at_one["probability_status"] + assert "1 field" in status_one + assert "constrained-path probability" in status_one + assert "uncalibrated" in status_one at_half = run_parallel_generation(engine, "ctx", schema, temperature=0.5) - assert "temperature=0.5" in at_half["probability_status"] - assert "temperature-scaled" in at_half["probability_status"] + status_half = at_half["probability_status"] + assert "T=0.5" in status_half + assert "temperature-scaled" in status_half @pytest.mark.slow diff --git a/tests/test_engine_fake.py b/tests/test_engine_fake.py index 64f7249..1d5c87b 100644 --- a/tests/test_engine_fake.py +++ b/tests/test_engine_fake.py @@ -64,9 +64,11 @@ def test_prompt_sha256_stable_and_input_sensitive(): assert len(r1["prompt_sha256"]) == 64 # Independent of the schema contents swap? No: same schema, so identical. assert r1["prompt_version"] == "jevmlx-parallel-v9" - assert ( - r1["probability_status"] - == "constrained-path probability at T=1; uncalibrated as decision confidence" + # W5b-13: status = per-group semantics summary (the fake ties -> + # rescored_batch1). + assert r1["probability_status"] == ( + "1 field: rescored_batch1; constrained-path probability; " + "uncalibrated as decision confidence" ) @@ -588,16 +590,18 @@ def test_probability_status_truthful_at_temperature_ne_one(): {"action": {"type": "enum", "description": "d", "choices": ["A", "B"]}} ) at_one = run_parallel_generation(make_engine(model, tokenizer), "ctx", schema, temperature=1.0) + # W5b-13: the status is the per-group summary. The zero-logit fake ties + # inside the band, so its single field's evidence is rescored_batch1. assert at_one["probability_status"] == ( - "constrained-path probability at T=1; uncalibrated as decision confidence" + "1 field: rescored_batch1; constrained-path probability; " + "uncalibrated as decision confidence" ) at_half = run_parallel_generation(make_engine(model, tokenizer), "ctx", schema, temperature=0.5) status = at_half["probability_status"] assert "temperature-scaled" in status - assert "temperature=0.5" in status - assert "not a T=1 probability" in status + assert "T=0.5" in status at_two = run_parallel_generation(make_engine(model, tokenizer), "ctx", schema, temperature=2.0) - assert "temperature=2.0" in at_two["probability_status"] + assert "T=2.0" in at_two["probability_status"] class _StatefulFakeCache: diff --git a/tests/test_field_semantics.py b/tests/test_field_semantics.py index 1b891b8..f652a0d 100644 --- a/tests/test_field_semantics.py +++ b/tests/test_field_semantics.py @@ -1,16 +1,16 @@ -"""W5b-13 step 1: FieldSemantics contract tests (design-note C9). - -The dataclass lives on the public API NOW; the ENGINE fills it in step 2 -(set in the #50 stages: score_scalar_field / score_multi_field / -reconcile_case_constraints / run_dependency_waves / finalize_public_result). -Until step 2 lands, decide() cannot supply semantics — the end-to-end -pinning test is xfail(strict=True) so it fails loudly today and flips the -suite green exactly when step 2 removes the mark. +"""W5b-13: per-field probability semantics (GPT-REVIEW-2 §C item 9). + +The engine's stages set a ``semantics`` record on every field_telemetry +entry (score_source / temperature / calibrator_id / prior_mode / +constraint_changed / dependency_rescored); the public API coerces it into +the frozen :class:`jevmlx.api.FieldSemantics`; the result-level +``probability_status`` summarizes the distinct semantic groups. """ from __future__ import annotations import dataclasses +import types from typing import Literal import pytest @@ -18,12 +18,13 @@ import jevmlx from jevmlx.api import FieldResult, FieldSemantics +from jevmlx.engine import run_parallel_generation # --- the dataclass contract ---------------------------------------------------- def test_field_semantics_frozen_and_field_names(): - """Frozen dataclass; the six C9 field names in the design order.""" + """Frozen dataclass; the six §C9 field names in the design order.""" s = FieldSemantics( score_source="batched", temperature=1.0, @@ -81,98 +82,301 @@ def test_field_semantics_exported(): assert "FieldSemantics" in api.__all__ -# --- engine-filling pinning test (step 2) -------------------------------------- +# --- engine filling: one pin per behavior --------------------------------------- + + +def _decide_with(monkeypatch, *, fields: dict, model_fields: type | None = None, **request_kwargs): + """decide() over the shared fake engine with the given telemetry fields. + + ``model_fields``: the pydantic model to decide against — defaults to a + single-scalar Ticket; pass a custom model when the fixture carries + other fields.""" + import jevmlx.api as api + from tests.conftest import make_engine_result + + result = make_engine_result(fields=fields) + monkeypatch.setattr(api, "run_parallel_generation", lambda *a, **k: result) + monkeypatch.setattr(api, "load_engine", lambda model_id: ("engine", "tokenizer")) + + if model_fields is None: + + class Ticket(BaseModel): + risk_tier: Literal["HIGH", "LOW", "CRITICAL"] = Field(description="Risk tier") + + model_fields = Ticket + + return jevmlx.decide(model_fields, "ctx", model="fake/model", **request_kwargs), result + + +def _scalar_field(**overrides): + from tests.conftest import make_field_semantics, make_field_telemetry + + return make_field_telemetry( + value="HIGH", + choices=["HIGH", "LOW", "CRITICAL"], + semantics=make_field_semantics(**overrides), + ) + + +def test_decide_fields_carry_semantics(monkeypatch): + """Every decided field carries a complete frozen FieldSemantics — the + engine's record is coerced at the public boundary; None is impossible. + + Both FieldSemantics bindings come from ONE import path inside the test + (the test_check_results module eviction can leave this file's + collection-time class object on a stale module instance — compare via + the same-path binding, not collection-time identity).""" + from jevmlx.api import FieldSemantics as FS # noqa: N806 — same-path binding + + d, _ = _decide_with( + monkeypatch, + fields={ + "risk_tier": _scalar_field(score_source="rescored_batch1", temperature=0.5), + }, + ) + sem = d.fields["risk_tier"].semantics + assert isinstance(sem, FS) + # Same-path binding for the equality too (cross-instance equality is by + # value here, but the constructor must come from the same module as the + # coercion — see the module-eviction note above). + assert dataclasses.asdict(sem) == dataclasses.asdict( + FS( + score_source="rescored_batch1", + temperature=0.5, + calibrator_id=None, + prior_mode="off", + constraint_changed=False, + dependency_rescored=False, + ) + ) -def _run_pinned_decide(monkeypatch): - """decide() over the shared fake engine with one scalar field: the - minimal path whose FieldResult must carry a complete semantics record - once the engine sets it (W5b-13 step 2).""" +def test_missing_semantics_record_fails_loudly(monkeypatch): + """A telemetry entry without a semantics record is a contract violation: + _build_field_results raises naming the field — never a silent None.""" import jevmlx.api as api from tests.conftest import make_engine_result, make_field_telemetry - monkeypatch.setattr( - api, - "run_parallel_generation", - lambda *a, **k: make_engine_result( - fields={ - "risk_tier": make_field_telemetry( - value="HIGH", - choices=["HIGH", "LOW", "CRITICAL"], - ) - }, - ), + result = make_engine_result( + fields={"risk_tier": make_field_telemetry(value="HIGH", choices=["HIGH", "LOW"])} ) + del result["field_telemetry"]["risk_tier"]["semantics"] + monkeypatch.setattr(api, "run_parallel_generation", lambda *a, **k: result) monkeypatch.setattr(api, "load_engine", lambda model_id: ("engine", "tokenizer")) class Ticket(BaseModel): - risk_tier: Literal["HIGH", "LOW", "CRITICAL"] = Field(description="Risk tier") - - return jevmlx.decide(Ticket, "ctx", model="fake/model") - - -@pytest.mark.xfail( - strict=True, - reason="W5b-13 step 2: the engine does not set FieldResult.semantics yet " - "(lands after the '#51 merged' ping; #50 stages set it per the design table)", -) -def test_engine_fills_semantics_per_field(monkeypatch): - """PIN (step 2): the engine's decided fields carry per-field semantics — - scalar path: batched evidence, the caller temperature, no calibrator, - prior off, no constraint change, no dependency rescore.""" - d = _run_pinned_decide(monkeypatch) - fr = d.fields["risk_tier"] - assert isinstance(fr.semantics, FieldSemantics) - assert fr.semantics == FieldSemantics( - score_source="batched", + risk_tier: Literal["HIGH", "LOW"] = Field(description="Risk tier") + + with pytest.raises(ValueError, match="semantics record"): + jevmlx.decide(Ticket, "ctx", model="fake/model") + + +def _finalize_with(fields: dict[str, dict]) -> dict: + """finalize_public_result over a typed AssembledState carrying the given + field_telemetry — the stage that builds probability_status.""" + from jevmlx.engine import Ledger, ScoreRowsResult, finalize_public_result + from jevmlx.schema import StructuredSchema + + schema = StructuredSchema( + { + "risk_tier": {"type": "enum", "description": "d", "choices": ["HIGH", "LOW"]}, + "tags": {"type": "multi", "description": "d", "choices": ["a", "b"]}, + } + ) + state = _AssembledStateShim(fields) + scored = ScoreRowsResult(row_logits={}, row_legal_mass_log={}, passes=1, chunk_shapes=[]) + built = {"scoring": "slots", "rows": []} + return finalize_public_result( + schema=schema, + state=state, # type: ignore[arg-type] + scored=scored, + built=built, + second_pass_telemetry={"rerun_fields": [], "rerun_rows": 0}, + timings={"peak_active_bytes": 1, "peak_incremental_bytes": 1}, temperature=1.0, - calibrator_id=None, - prior_mode="off", - constraint_changed=False, - dependency_rescored=False, + prior_correction=False, + prior_ms=0.0, + constraints=None, + base_ids=[], + active_start=0, + ledger=Ledger(), ) -@pytest.mark.xfail( - strict=True, - reason="W5b-13 step 2: probability_status stays the old global string until " - "the stages set semantics and finalize_public_result builds the summary", -) -def test_probability_status_is_a_summary(monkeypatch): - """PIN (step 2, written cleanly against the then-existing behavior): the - result-level probability_status is a SUMMARY over the distinct - (score_source, temperature, calibrator_id, prior_mode) groups — field - counts per group, no per-field probability claims. Replaces the - placeholder draft deleted in review (F2).""" - d = _run_pinned_decide(monkeypatch) - assert isinstance(d.fields["risk_tier"].semantics, FieldSemantics) - status = _engine_probability_status(monkeypatch) - # One distinct group (scalar, T=1, uncalibrated, prior off): the summary - # mentions that group with its field count and no per-field values. - assert "1 field" in status +class _AssembledStateShim: + """Duck-typed AssembledState: finalize_public_result reads .field_telemetry + plus the pass-through tuples; a real AssembledState requires frozen dict + shapes the tests don't need.""" + + def __init__(self, field_telemetry): + self.field_telemetry = field_telemetry + self.rescored_fields = () + self.reconciled_fields = () + self.internal_telemetry = types.MappingProxyType({}) + self.parsed_json = { + name: {"value": ft.get("value")} for name, ft in field_telemetry.items() + } + + +def test_probability_status_is_a_group_summary(): + """The result-level probability_status is a SUMMARY over the distinct + (score_source, temperature, calibrator_id, prior_mode) groups: one clause + per group with its field count — not authoritative for any single field.""" + from tests.conftest import make_field_semantics, make_field_telemetry + + result = _finalize_with( + { + # Two fields share one group (scalar, T=1, uncalibrated, prior off)... + "risk_tier": make_field_telemetry( + value="HIGH", choices=["HIGH", "LOW"], semantics=make_field_semantics() + ), + "tags": make_field_telemetry( + value=["a"], + type_="multi", + choices=["a", "b"], + per_option={"a": 0.9, "b": 0.1}, + semantics=make_field_semantics(temperature=None), + ), + } + ) + status = result["probability_status"] + # ...but here they differ (multi carries temperature=None) -> 2 groups. + assert status.count("1 field:") == 2 + # Group clauses name the semantics dimensions. assert "batched" in status - assert "T=1" in status + assert "prior-corrected" not in status # both groups are prior_mode=off -def _engine_probability_status(monkeypatch) -> str: - """The raw engine result's probability_status through the fake path.""" - import jevmlx.api as api - from tests.conftest import make_engine_result, make_field_telemetry +def test_probability_status_names_prior_and_calibrator_groups(): + """A prior-corrected scalar and a calibrated multi land in DISTINCT + groups; each clause carries its own prior_mode / calibrator identity.""" + from tests.conftest import make_field_semantics, make_field_telemetry - res = make_engine_result( - fields={ + result = _finalize_with( + { "risk_tier": make_field_telemetry( value="HIGH", - choices=["HIGH", "LOW", "CRITICAL"], - ) - }, + choices=["HIGH", "LOW"], + semantics=make_field_semantics(prior_mode="neutral_v1"), + ), + "tags": make_field_telemetry( + value=["a"], + type_="multi", + choices=["a", "b"], + per_option={"a": 0.9, "b": 0.1}, + semantics=make_field_semantics(temperature=None, calibrator_id="test-rev-abc"), + ), + } ) - monkeypatch.setattr(api, "run_parallel_generation", lambda *a, **k: res) - monkeypatch.setattr(api, "load_engine", lambda model_id: ("engine", "tokenizer")) + status = result["probability_status"] + assert "prior-corrected against the neutral-context pass" in status + assert "calibrated (bundle test-rev-abc)" in status + + +# --- engine-stage pins (real fake-engine path, one behavior each) ---------------- + + +def test_engine_sets_semantics_on_all_shapes(): + """The engine's stages set a semantics record on EVERY telemetry shape: + scalar, multi, cardinality-1, and the dependency re-decided child.""" + from jevmlx.schema import StructuredSchema + from tests.conftest import FakeModel, FakeTokenizer, make_engine + from tests.test_w5b import BijectiveTokenizer, RoutingBiasModel, _chain_schema + + # Scalar + multi (FakeModel ties -> rescored_batch1 on both). + schema = StructuredSchema( + { + "action": {"type": "enum", "description": "d", "choices": ["A", "B"]}, + "flags": {"type": "multi", "description": "d", "choices": ["x", "y"]}, + } + ) + r = run_parallel_generation(make_engine(FakeModel(), FakeTokenizer()), "ctx", schema) + sem_action = r["field_telemetry"]["action"]["semantics"] + assert sem_action["score_source"] == "rescored_batch1" + assert sem_action["temperature"] == 1.0 + assert sem_action["prior_mode"] == "off" + # Multi: temperature None (fake ties are rescored; uncalibrated -> caller + # T applies to P(yes) softmax... T=1.0 here). + sem_flags = r["field_telemetry"]["flags"]["semantics"] + assert sem_flags["score_source"] == "rescored_batch1" + + # Cardinality-1 + dependency re-decided child (chain schema: pa has one + # choice -> schema-determined; cb re-decided in a wave). + chain = StructuredSchema(_chain_schema()) + r2 = run_parallel_generation( + make_engine(RoutingBiasModel(), BijectiveTokenizer()), "ctx", chain + ) + sem_pa = r2["field_telemetry"]["pa"]["semantics"] + assert sem_pa["score_source"] == "batched" + assert sem_pa["temperature"] is None # schema-determined: no T applied + sem_cb = r2["field_telemetry"]["cb"]["semantics"] + assert sem_cb["score_source"] == "dependency" + assert sem_cb["dependency_rescored"] is True + + +def test_constraint_flip_records_constraint_changed(): + """A case-constraint MAP flip sets constraint_changed=True on exactly the + flipped field's semantics record.""" + from jevmlx.schema import StructuredSchema + from tests.conftest import make_engine + from tests.test_w5b import BijectiveTokenizer, RoutingBiasModel + + schema = StructuredSchema( + { + "a": {"type": "enum", "description": "d", "choices": ["A", "B"]}, + "b": {"type": "enum", "description": "d", "choices": ["X", "Y"]}, + } + ) + # Both fields bias to their first choice (A, X); A and X are forbidden + # jointly, so the MAP must flip one of them. + cons = [{"type": "excludes", "field": "a", "value": "A", "other": "b", "other_value": "X"}] + r = run_parallel_generation( + make_engine(RoutingBiasModel(), BijectiveTokenizer()), "ctx", schema, constraints=cons + ) + assert r["reconciled_fields"] + for fname in r["reconciled_fields"]: + assert r["field_telemetry"][fname]["semantics"]["constraint_changed"] is True + untouched = set(r["field_telemetry"]) - set(r["reconciled_fields"]) + for fname in untouched: + assert r["field_telemetry"][fname]["semantics"]["constraint_changed"] is False + + +def test_calibrated_multi_and_scalar_temperature_split_groups(): + """A calibrated multi ignores the caller temperature (record temperature + None + bundle id); the scalar next to it carries the caller T — two + distinct groups in the summary.""" + from jevmlx.calibrate import CalibrationBundle + from jevmlx.schema import StructuredSchema + from tests.conftest import FakeModel, FakeTokenizer, make_engine + + schema = StructuredSchema( + { + "action": {"type": "enum", "description": "d", "choices": ["A", "B"]}, + "flags": {"type": "multi", "description": "d", "choices": ["x", "y"]}, + } + ) + bundle = CalibrationBundle.from_payload({"multi": {"a": 1.0, "b": 0.0}}) + r = run_parallel_generation( + make_engine(FakeModel(), FakeTokenizer()), + "ctx", + schema, + temperature=0.7, + calibration=bundle, + ) + sem_flags = r["field_telemetry"]["flags"]["semantics"] + assert sem_flags["temperature"] is None + assert sem_flags["calibrator_id"] == bundle.identity() + sem_action = r["field_telemetry"]["action"]["semantics"] + assert sem_action["temperature"] == 0.7 + assert sem_action["calibrator_id"] is None + # The summary carries both groups separately. + status = r["probability_status"] + assert "T=0.7" in status + assert f"calibrated (bundle {bundle.identity()})" in status + assert "temperature not applied (fixed rule selection)" in status - class Ticket(BaseModel): - risk_tier: Literal["HIGH", "LOW", "CRITICAL"] = Field(description="Risk tier") - d = jevmlx.decide(Ticket, "ctx", model="fake/model") - assert d.fields["risk_tier"].semantics is not None # step-2 gate - return res["probability_status"] +def test_run_parallel_generation_imported_here(): + """Guard: the module-level import used by the pins above resolves to the + live engine module (the test_check_results eviction trap).""" + from jevmlx.engine import run_parallel_generation # noqa: F401 From 81572abb5c19a963a04ce67f5eed37d733b4c08b Mon Sep 17 00:00:00 2001 From: Ben Shaharizad Date: Sat, 19 Sep 2026 12:06:19 +0300 Subject: [PATCH 4/4] W5b-13 review round 2: blockers F1-F2, shoulds F3-F4/F6/F8-F9, nits F5/F7/F10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - F1: the old global-only probability_status strings are DELETED; an empty semantics-group set now raises (every engine path sets records). - F2: all four None-tolerant sites index directly — finalize_public_result (ft['semantics']), reconcile_case_constraints, the post-dependency MAP, _selective_second_pass (prev record + assignment), api boundary (telemetry['semantics']). grep 'get("semantics")' jevmlx/*.py is EMPTY. - F3: count-row constraint_changed = reconciled_by == 'count' AND the trusted count actually ran (dropped_reason None); an untrusted count no longer claims a constraint. - F4: multi constraint_changed compares the FINAL selection against the raw threshold proposal (solve_multi_set now records telemetry['threshold_proposal']); a trusted non-binding count no longer claims a change. - F5: score_scalar_field no longer re-derives the applied temperature — the temperature argument IS the effective one (_load_calibration replaced it); cardinality-1 outcome built via _field_semantics. - F6: three new engine-path pins (prior_correction -> neutral_v1 on a real run; count-row temperature None + copied score_source; uncalibrated multi carries caller T) + trusted/untrusted-count constraint_changed pin — each mutation-verified to fail when its setter is removed. - F7: contradictory comment in test_engine_sets_semantics_on_all_shapes fixed (multi carries caller T=1.0 when uncalibrated). - F8: CHANGELOG Unreleased entry for W5b-13. - F9: ARCHITECTURE.md — probability_status row = group summary; 'semantics' row in the field_telemetry table and the FieldResult table. - F10: FieldResult.semantics docstring no longer says 'None until then'. 750 passed -m 'not slow', slow status pin green, ruff clean. --- ARCHITECTURE.md | 4 +- CHANGELOG.md | 17 ++++++ jevmlx/api.py | 15 +++-- jevmlx/engine.py | 101 +++++++++++++++++----------------- tests/test_field_semantics.py | 94 +++++++++++++++++++++++++++++-- 5 files changed, 168 insertions(+), 63 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 621ebfb..523ebc6 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -207,7 +207,7 @@ load). | `schema_match` | Always True (keys/enums guaranteed by construction). | | `confidence_model` | `"slots"` or `"labels"`. | | `prompt_sha256` / `prompt_version` | SHA-256 over the full prompt token ids; the version is read from `engine.PROMPT_VERSION` (v8) — never a literal elsewhere. | -| `probability_status` | How to read the probabilities. | +| `probability_status` | W5b-13: a SUMMARY over the per-field semantics records — one clause per distinct (score_source, temperature, calibrator_id, prior_mode) group with its field count. Not authoritative for any single field; the per-field truth is `field_telemetry[..]['semantics']`. An empty group set raises (every engine path sets records). | | `prior_correction` / `constraints_applied` | Whether the prior pass ran / case-level constraints were applied. | | `reconciled_fields` | Fields whose value changed under constrained MAP. | | `parsed_json` | `{field: {"value": …, "prob": …}}`. | @@ -235,6 +235,7 @@ Batched-only keys (`run_parallel_generation_batched`, every result): `group_wall | `rows` | Rows the field consumed (0 for cardinality-1 fields). | | `tie` / `rescored` | Scalar: whether the top-2 gap is inside `INSTABILITY_BAND` (subsumes exact-equality ties), and whether the batch=1 rescore replaced the batched result. Multi: `rescored` when any option's Y/N pair was rescored. | | `evidence_source` | Scalar only (W5-B, PR #45): which scoring path produced the final evidence — `batch` (batched pass), `batch1` (canonical rescore replaced it), `dependency` (second pass), `oracle` (forced re-score). Rides on scalar entries and on second-pass re-decides. | +| `semantics` | W5b-13: the per-field semantics record (coerced to frozen `api.FieldSemantics` at the public boundary): `score_source` (`batched`/`rescored_batch1`/`dependency`/`oracle`), `temperature` actually applied (None for count rows and calibrated multi selections), `calibrator_id` (bundle identity when a fitted calibrator set the selection), `prior_mode` (`off`/`neutral_v1`), `constraint_changed` (a reconciler overrode the raw winner — an actual selection change, not merely a binding constraint), `dependency_rescored`. REQUIRED on every entry — `_build_field_results` raises without it. | | `legal_mass` | Probability the model assigned to the union of allowed continuations at the winner's branch point(s), against the full vocabulary = sum(exp(z_allowed)) / sum(exp(z_vocab)). Per-branch leakage signal — the constrained distribution can confidently pick A over B even when almost all unconstrained mass is on a reasoning token/newline/label text. Product over the winner's branch path (scalar); per-option Y/N branches (multi); the count row has its own (`legal_mass` + `min_option_legal_mass` on the `#count` entry). 1.0 for cardinality-1 fields (nothing branched). Always computed. Raw, pre-prior-correction logits. | | `legal_mass_logs` | Per-choice (scalar) / per-option (multi) natural-log legal-mass product along the branch path, keyed by the real choice/option string. Raw, T=1. Calibration feature for the abstention model. The legal-mass callback is LOG-space end to end (W5-D finding 37: `score_trie`'s `legal_mass_at_node` returns natural-log floats; the trie stays MLX-free). | | `min_option_legal_mass` / `mean_log_legal_mass` | Multi only (W5-D finding 38): cardinality-free field-level stats replacing the old underflowing, cardinality-confounded product as the headline numbers — the worst option's legal mass in probability space, and the mean per-option log mass (additive, stable). The per-option logs stay on `legal_mass_logs`. | @@ -255,6 +256,7 @@ Batched-only keys (`run_parallel_generation_batched`, every result): `group_wall | `model` | `"slots"` or `"labels"`. | | `alternatives` | Top 3 (choice, probability) pairs; multi: per-option (option, P(yes)) sorted desc. | | `reason` | None, `"none_of_above"` (caller opted in via `allow_none_of_above=True`, model picked the explicit opt-out → None), or `"abstain"` (`abstain_below_margin` set and the field's margin — `probability_margin` scalar / `threshold_distance` multi — fell below the cut; value withheld from the validated instance, raw kept for provenance). The single source of truth — no separate abstain flag. | +| `semantics` | Frozen `api.FieldSemantics` (W5b-13): how THIS field's reported probabilities were produced — score_source, the temperature actually applied, the calibrator bundle id, prior_mode, constraint_changed, dependency_rescored. Required, kw-only; coerced from the telemetry record, never None. | ### Case-level constraints — `constraints.py`, applied by `_constrained_map` diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f8ea2a..f7b8761 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -235,6 +235,23 @@ (per-row peak-memory slope per width bin under `mx.reset_peak_memory`, B=1/2/4/8 at widths 4/8/16/32) and `adapters` (max abs diff + timing of full-width head vs decision-position head). No engine wiring. +- Per-field probability semantics (W5b-13): every engine scoring stage — + scalar finalization, multi selection, count rows, case-constraint MAP, + dependency waves — sets a frozen semantics record + (`api.FieldSemantics`: score_source / temperature / calibrator_id / + prior_mode / constraint_changed / dependency_rescored) on its + field_telemetry entry, and `FieldResult.semantics` (required, kw-only) + coerces it at the public API boundary — a missing record raises, so + 'required' holds at every decide() return. The result-level + `probability_status` is now a SUMMARY over the distinct + (score_source, temperature, calibrator_id, prior_mode) groups — one + clause per group with its field count; the old global-only statement is + gone (a result mixing temperature-scaled scalars, calibrated multi + options and dependency re-scores cannot be described by one sentence). + A calibrated multi records temperature None (the log-odds cut ignores + the caller T); count rows always do (fixed T=1 buckets); + constraint_changed reflects an actual selection change, not merely a + binding constraint. - Results contract v2 in the results tooling (PR #48): `check_results.py` requires `timing.json` on parallel-track combos with the full timing-split median (incl. `peak_incremental_bytes` + diff --git a/jevmlx/api.py b/jevmlx/api.py index fbcd3bd..71cda7a 100644 --- a/jevmlx/api.py +++ b/jevmlx/api.py @@ -159,11 +159,9 @@ class FieldResult: only values are None, "none_of_above" and "abstain"; a withheld decision is exactly ``reason == "abstain"``. semantics: HOW this field's reported probabilities were produced — - see :class:`FieldSemantics`. Required: the engine fills it in - W5b-13 step 2 (this draft's decide() passes - telemetry.get("semantics") through, which is None until then; - the PR does not leave draft until every decided field carries a - real record). + see :class:`FieldSemantics`. Required: the engine's stages set + the record and _build_field_results coerces it; a telemetry + entry without one raises (never a silent None). """ value: object @@ -406,9 +404,10 @@ def _build_field_results( # W5b-13 step 2: the engine's stages set the semantics record; the # public API coerces it into the frozen FieldSemantics. A telemetry # entry without one is a contract violation — fail loudly, never - # ship a None semantics (the docstring's 'required' is now TRUE at - # every decide() return). - sem_dict = telemetry.get("semantics") + # ship a None semantics. (This is the boundary CHECK, not a + # tolerant read: the index below raises when the record is absent + # or not a dict.) + sem_dict = telemetry["semantics"] if not isinstance(sem_dict, dict): raise ValueError( f"field telemetry for {name!r} carries no semantics record " diff --git a/jevmlx/engine.py b/jevmlx/engine.py index 88a13b2..a497053 100644 --- a/jevmlx/engine.py +++ b/jevmlx/engine.py @@ -1957,8 +1957,9 @@ def mass_at_node(node: dict, _l=branch_mass, _i=branch_index) -> float: # A MAP reconciliation that happened EARLIER stays recorded # (constraint_changed carries over); the oracle path records # without touching the main predictions, so its semantics live - # only on the fields it re-decided. - _prev_sem = field_telemetry[fname].get("semantics") or {} + # only on the fields it re-decided. F2: the record must exist — + # index directly. + _prev_sem = field_telemetry[fname]["semantics"] field_telemetry[fname]["semantics"] = _field_semantics( score_source="oracle" if is_oracle else "dependency", temperature=temperature, @@ -1992,9 +1993,9 @@ def mass_at_node(node: dict, _l=branch_mass, _i=branch_index) -> float: if fname in parsed_json and parsed_json[fname]["value"] != val: parsed_json[fname]["value"] = val field_telemetry[fname]["value"] = val - # W5b-13: the post-dependency MAP changed this field again. - if "semantics" in field_telemetry[fname]: - field_telemetry[fname]["semantics"]["constraint_changed"] = True + # W5b-13: the post-dependency MAP changed this field again + # (F2: the record must exist — index directly). + field_telemetry[fname]["semantics"]["constraint_changed"] = True final_assignment = {fname: pj["value"] for fname, pj in parsed_json.items()} if not compiled_constraints.satisfied(final_assignment): raise InternalConstraintViolationError( @@ -2692,15 +2693,15 @@ def _cardinality_one_outcome( "rows": 0, "legal_mass": 1.0, # W5b-13: cardinality-1 fields are schema-determined — no model - # scoring, no temperature applied, nothing corrected. - "semantics": { - "score_source": "batched", - "temperature": None, - "calibrator_id": None, - "prior_mode": "off", - "constraint_changed": False, - "dependency_rescored": False, - }, + # scoring, no temperature applied, nothing corrected (F5: built + # by the ONE record constructor). + "semantics": _field_semantics( + score_source="batched", + temperature=None, + calib=None, + calibrated_applied=False, + prior_corrected=False, + ), }, ) @@ -2792,15 +2793,12 @@ def score_scalar_field( ) calibration_id = calib.identity() if scalar_calibrated is not None else None # W5b-13: the per-field semantics record. Evidence source from the ONE - # finalizer (batch -> batched, batch1 -> rescored_batch1); the applied - # temperature is the fitted scalar T when the bundle supplied it, the - # caller temperature otherwise. - applied_temperature = ( - calib.temperature if calib is not None and calib.has_scalar else temperature - ) + # finalizer (batch -> batched, batch1 -> rescored_batch1). F5: the + # temperature arriving here IS the effective one — _load_calibration + # already replaced the caller T with the bundle's fitted scalar T. semantics = _field_semantics( score_source="rescored_batch1" if decision.rescored else "batched", - temperature=applied_temperature, + temperature=temperature, calib=calib, calibrated_applied=scalar_calibrated is not None, prior_corrected=decision.prior_corrected, @@ -3026,6 +3024,10 @@ def solve_multi_set( telemetry = { "margin": margin, "reconciled_by": reconciled_by, + # W5b-13 F4: the raw threshold proposal — what constraint_changed + # compares the final selection against (reconciled_by == 'count' + # also fires when a trusted count was non-binding). + "threshold_proposal": sorted(selected_set), # W2-E step 3: the count row's answer and confidence. "count_choice": count_choice, "count_margin": count_margin, @@ -3294,6 +3296,12 @@ def score_multi_field( prior_pairs, selected, ) + # W5b-13 F4: constraint_changed compares the FINAL selected set to the + # raw threshold proposal (solved_telemetry['threshold_proposal']) — + # reconciled_by == 'count' also fires when a trusted count was + # non-binding (the solver reproduced the proposal), and that changed + # nothing. + constraint_changed = set(selected) != set(solved_telemetry["threshold_proposal"]) # W5b-13: multi semantics. score_source mirrors the scalar vocabulary — # 'rescored_batch1' when any option's Y/N pair was band-rescored. The # caller temperature was applied to the P(yes) softmax, BUT the @@ -3307,14 +3315,14 @@ def score_multi_field( calib=calib, calibrated_applied=calib is not None and calib.has_multi, prior_corrected=prior_entry is not None, - # solve_multi_set's reconciled_by: 'per_option' = the raw threshold - # proposal stood; 'count' or a setcons rule name = a reconciler - # overrode it (W5b-13 constraint_changed). - constraint_changed=solved_telemetry.get("reconciled_by") != "per_option", + constraint_changed=constraint_changed, ) # The count row rides the parent multi's prior mode and score path; its # bucket scores are fixed T=1 softmaxes (never temperature-scaled) and - # its selection is the trusted-count rule (a reconciler by nature). + # its selection is the trusted-count rule. F3: constraint_changed only + # when a trusted count constraint actually RAN — an untrusted count + # (margin below COUNT_MARGIN_MIN) never became a constraint, and + # dropped_reason stays None on that path too. if count_telemetry is not None: count_telemetry["semantics"] = _field_semantics( score_source="rescored_batch1" if multi_rescored else "batched", @@ -3322,7 +3330,8 @@ def score_multi_field( calib=None, calibrated_applied=False, prior_corrected=prior_entry is not None, - constraint_changed=count_telemetry.get("dropped_reason") is None, + constraint_changed=solved_telemetry.get("reconciled_by") == "count" + and count_telemetry.get("dropped_reason") is None, ) return FieldOutcome( fname, @@ -3381,9 +3390,8 @@ def reconcile_case_constraints( field_log_scores[fname][str(val)] ) # W5b-13: the MAP overrode the raw winner — the field's - # semantics record says so. - if "semantics" in field_telemetry[fname]: - field_telemetry[fname]["semantics"]["constraint_changed"] = True + # semantics record says so (F2: must exist — index directly). + field_telemetry[fname]["semantics"]["constraint_changed"] = True return AssembledState( parsed_json=parsed_json, field_telemetry=field_telemetry, @@ -3482,9 +3490,9 @@ def finalize_public_result( # re-scores cannot be described by one sentence. semantic_groups: dict[tuple, list[str]] = {} for fname, ft in state.field_telemetry.items(): - sem = ft.get("semantics") - if not isinstance(sem, dict): - continue + # F2: the record must exist — index directly. A missing record is a + # results-contract violation, never a skippable field. + sem = ft["semantics"] key = (sem["score_source"], sem["temperature"], sem["calibrator_id"], sem["prior_mode"]) semantic_groups.setdefault(key, []).append(fname) clauses: list[str] = [] @@ -3510,23 +3518,16 @@ def finalize_public_result( if pmode == "neutral_v1": parts.append("prior-corrected against the neutral-context pass") clauses.append("; ".join(parts)) - if clauses: - probability_status = " per distinct semantics group | ".join(clauses) - else: - # No semantics records (fake/baseline paths): keep the classic - # temperature-honest statement rather than an empty string. - if temperature == 1.0: - probability_status = ( - "constrained-path probability at T=1; uncalibrated as decision confidence" - ) - else: - probability_status = ( - f"post-hoc temperature-scaled constrained distribution " - f"(temperature={temperature}); ranking-invariant, not a T=1 probability; " - f"uncalibrated as decision confidence" - ) - if prior_correction: - probability_status += "; prior-corrected against the neutral-context pass" + # Every engine path sets semantics records (F1): an empty group set is + # a bug, not a fallback case — fail loudly instead of shipping a global + # sentence that could contradict the per-field records. + if not clauses: + raise ValueError( + "finalize_public_result: no field carries a semantics record " + "(results-contract violation; the stages must set " + "field_telemetry[fname]['semantics'])" + ) + probability_status = " per distinct semantics group | ".join(clauses) # Bug 9 / W5b-14: the timing split is honest about the whole request # wall time and every key is a ledger derivation (prior_ms = the prior diff --git a/tests/test_field_semantics.py b/tests/test_field_semantics.py index f652a0d..6f03578 100644 --- a/tests/test_field_semantics.py +++ b/tests/test_field_semantics.py @@ -153,7 +153,9 @@ def test_decide_fields_carry_semantics(monkeypatch): def test_missing_semantics_record_fails_loudly(monkeypatch): """A telemetry entry without a semantics record is a contract violation: - _build_field_results raises naming the field — never a silent None.""" + _build_field_results raises naming the field — never a silent None + (F2: the boundary indexes directly; a KeyError and the ValueError are + both loud, so the pin accepts either).""" import jevmlx.api as api from tests.conftest import make_engine_result, make_field_telemetry @@ -167,7 +169,7 @@ def test_missing_semantics_record_fails_loudly(monkeypatch): class Ticket(BaseModel): risk_tier: Literal["HIGH", "LOW"] = Field(description="Risk tier") - with pytest.raises(ValueError, match="semantics record"): + with pytest.raises((ValueError, KeyError), match="semantics"): jevmlx.decide(Ticket, "ctx", model="fake/model") @@ -295,10 +297,12 @@ def test_engine_sets_semantics_on_all_shapes(): assert sem_action["score_source"] == "rescored_batch1" assert sem_action["temperature"] == 1.0 assert sem_action["prior_mode"] == "off" - # Multi: temperature None (fake ties are rescored; uncalibrated -> caller - # T applies to P(yes) softmax... T=1.0 here). + # Multi: uncalibrated fake run — the caller T (1.0) IS applied to the + # P(yes) softmax, so the record carries it (None only under calibration). sem_flags = r["field_telemetry"]["flags"]["semantics"] assert sem_flags["score_source"] == "rescored_batch1" + assert sem_flags["temperature"] == 1.0 + assert sem_flags["prior_mode"] == "off" # Cardinality-1 + dependency re-decided child (chain schema: pa has one # choice -> schema-determined; cb re-decided in a wave). @@ -380,3 +384,85 @@ def test_run_parallel_generation_imported_here(): """Guard: the module-level import used by the pins above resolves to the live engine module (the test_check_results eviction trap).""" from jevmlx.engine import run_parallel_generation # noqa: F401 + + +def test_prior_correction_lands_neutral_v1_on_real_run(): + """F6a: prior_correction=True on a REAL engine path lands + prior_mode='neutral_v1' on every field's semantics record — asserted on + the run, not a hand-made dict. Fails if the scalar/multi setters stop + reading decision.prior_corrected / prior_entry.""" + from jevmlx.schema import StructuredSchema + from tests.conftest import FakeModel, FakeTokenizer, make_engine + + schema = StructuredSchema( + { + "action": {"type": "enum", "description": "d", "choices": ["A", "B"]}, + "flags": {"type": "multi", "description": "d", "choices": ["x", "y"]}, + } + ) + corrected = run_parallel_generation( + make_engine(FakeModel(), FakeTokenizer()), "ctx", schema, prior_correction=True + ) + raw = run_parallel_generation(make_engine(FakeModel(), FakeTokenizer()), "ctx", schema) + for fname in ("action", "flags"): + assert corrected["field_telemetry"][fname]["semantics"]["prior_mode"] == "neutral_v1" + assert raw["field_telemetry"][fname]["semantics"]["prior_mode"] == "off" + + +def test_count_row_semantics_temperature_none_and_score_source_copied(): + """F6b: the count row's semantics carries temperature None (fixed T=1 + bucket softmaxes) and copies the parent multi's score_source + prior + mode. Fails if the count setter stops deriving from the parent.""" + from jevmlx.schema import StructuredSchema + from tests.conftest import FakeModel, FakeTokenizer, make_engine + + schema = StructuredSchema( + {"flags": {"type": "multi", "description": "d", "choices": ["x", "y"]}} + ) + r = run_parallel_generation( + make_engine(FakeModel(), FakeTokenizer()), "ctx", schema, prior_correction=True + ) + count = r["internal_telemetry"]["flags#count"] + parent = r["field_telemetry"]["flags"] + assert count["semantics"]["temperature"] is None + assert count["semantics"]["score_source"] == parent["semantics"]["score_source"] + assert count["semantics"]["prior_mode"] == parent["semantics"]["prior_mode"] == "neutral_v1" + + +def test_uncalibrated_multi_carries_caller_temperature(): + """F6c: an UNCALIBRATED multi records the caller temperature (the P(yes) + softmax used it) and prior_mode off without prior correction. Fails if + the multi setter stops passing the caller T on the uncalibrated path.""" + from jevmlx.schema import StructuredSchema + from tests.conftest import FakeModel, FakeTokenizer, make_engine + + schema = StructuredSchema( + {"flags": {"type": "multi", "description": "d", "choices": ["x", "y"]}} + ) + r = run_parallel_generation( + make_engine(FakeModel(), FakeTokenizer()), "ctx", schema, temperature=0.6 + ) + sem = r["field_telemetry"]["flags"]["semantics"] + assert sem["temperature"] == 0.6 + assert sem["calibrator_id"] is None + assert sem["prior_mode"] == "off" + + +def test_trusted_nonbinding_count_reports_constraint_changed_false(): + """F3/F4: neither an untrusted count (margin below COUNT_MARGIN_MIN: no + constraint ran) nor a trusted count whose constraint was non-binding + may claim constraint_changed — only a reconciler that CHANGED the + selection (or the count row's own trusted constraint running) may.""" + from jevmlx.schema import StructuredSchema + from tests.conftest import FakeModel, FakeTokenizer, make_engine + + schema = StructuredSchema( + {"flags": {"type": "multi", "description": "d", "choices": ["x", "y"]}} + ) + # Zero logits -> P(yes)=0.5 for every option; the count row ties across + # buckets (untrusted), the threshold proposal selects all, nothing + # changed anything. + r = run_parallel_generation(make_engine(FakeModel(), FakeTokenizer()), "ctx", schema) + sem = r["field_telemetry"]["flags"]["semantics"] + assert sem["constraint_changed"] is False + assert r["internal_telemetry"]["flags#count"]["semantics"]["constraint_changed"] is False