From ab73608541f0cf0f193db44bba584606fc8f6c4b Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 20 Jul 2026 12:02:10 +1000 Subject: [PATCH 01/10] docs: accumulator set-wildcard in-band sentinel redesign (A1) design spec Supersedes the null-list wildcard A1 approach. Represents set wildcards as [unknown_sentinel_for(dtype)] across input/filter/build; frontier fix is free (normalization makes co_ columns null-free + canonical, _frontier_filter untouched). Honors the null-is-not-a-portable-sentinel principle. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...ccumulator-set-sentinel-wildcard-design.md | 142 ++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-20-accumulator-set-sentinel-wildcard-design.md diff --git a/docs/superpowers/specs/2026-07-20-accumulator-set-sentinel-wildcard-design.md b/docs/superpowers/specs/2026-07-20-accumulator-set-sentinel-wildcard-design.md new file mode 100644 index 0000000..7d62509 --- /dev/null +++ b/docs/superpowers/specs/2026-07-20-accumulator-set-sentinel-wildcard-design.md @@ -0,0 +1,142 @@ +# Accumulator Set-Wildcard: In-Band Sentinel Redesign — Design Spec + +**Status:** Design (not yet implemented) +**Date:** 2026-07-20 +**Supersedes:** the `SET_MEMBERSHIP`/`SET_EXCLUSION` coalescing portion (A1) of `2026-07-19-accumulator-core-extensions-design.md`. The A2 aggregate-ops portion of that spec shipped independently (PR #51 → `develop`) and is unaffected. + +## Problem + +A1 set-coalescing encoded a set-dimension **wildcard as a null list** (matching mountainash's `t_is_in`, where a null rule list scores ternary 0). The whole-branch review found this breaks the accumulator build's frontier-dominance pruning: + +`_frontier_filter` (`engines/accumulator/engine.py`) removes dominated combinations via a self-join on the `co_`/`na_` fingerprint columns, which include the list-typed `co_`. For a set wildcard that column is **null**, and polars inner joins drop null keys (`join_nulls=False`, which mountainash's backend-agnostic `join` API does not expose — null-safe join is a polars-only flag, absent on narwhals and false-by-default in ibis/SQL). So any combination whose set dimension coalesced to a wildcard is **never detected as dominated and survives**. + +**Verified:** three all-wildcard `SET_MEMBERSHIP` rules produce combinations `{2,3,5,6,10,15,30}` instead of the single maximal `{30}`; `apply` returns 7 survivors instead of 1, with wrong accumulated aggregates. The identical shape with an EXACT-wildcard (concrete `UNKNOWN` sentinel) dimension correctly collapses to `{30}`. + +**Root cause:** using `null` as a *semantic marker*. Null behaves inconsistently across backends in exactly the operations where a marker is compared (join / group / membership). This violates the mountainash-central principle [`null-is-not-a-portable-sentinel`](../../../../mountainash-central/01.principles/mountainash/d.cross-backend/core/null-is-not-a-portable-sentinel.md). The fix is to represent the wildcard with an in-band typed **sentinel**, never a null — consistent with the scalar `UNKNOWN`/`NOT_SET` convention the rest of the engine already uses. + +## Goals + +- Set-dimension wildcards are represented by a non-null, in-band typed sentinel across **input, filter, and build** (scope B — a unified representation end to end). +- The frontier self-join dedupes wildcard-set combinations correctly; `apply` returns correct survivors and aggregates. +- No new native backend imports, no new `# allow:` tags; no mountainash change required. +- Backward-compatible: rules that express a wildcard as a null list are **normalized**, never rejected. +- The pre-existing set-op ordering nondeterminism (equal sets → distinct fingerprints) is fixed in the same stroke. + +## Non-Goals + +- Changing the A2 aggregate-ops behaviour (already shipped). +- Any change to `_frontier_filter`, `_check_overflow`, partition routing, or scalar/range/string strategies. +- Upstream mountainash changes (a separate backlog item 58 tracks removing the one `to_polars()` seam; not required here). + +## Canonical Representation + +A set-dimension **wildcard** is the single-element list `[unknown_sentinel_for(dim.data_type)]`: + +| data_type | wildcard list | +|---|---| +| str | `[""]` | +| int | `[-999999999]` | +| float | `[-999999999.0]`* | +| date | `[date(1,1,1)]` | +| datetime | `[datetime(1,1,1)]` | + +\* whatever `unknown_sentinel_for(float)` returns; the spec uses the existing `unknown_sentinel_for` — no new sentinels are introduced. + +The sentinel is a reserved, out-of-domain value — the same assumption the scalar `UNKNOWN` wildcard already relies on, so a concrete rule list never contains it. An **empty** concrete list `[]` keeps its concrete meaning (membership `[]` = matches nothing; exclusion `[]` = excludes nothing) and is cleanly distinct from `[sentinel]` — the empty-vs-wildcard ambiguity of the old design disappears. + +Feasibility verified: `ma.lit([sentinel])` produces a `List(element)`-typed literal; `when(is_null).then(lit([sentinel])).otherwise(col)` normalizes null → `[sentinel]` and preserves dtype; `col.list.contains(lit(sentinel))` detects it. All backend-pure (route through `mountainash.expressions`), no `# allow:` tag. + +## Shared Helpers (`core`) + +Three helpers live in a **new shared module `core/set_wildcard.py`** (they build `mountainash.expressions` and need `unknown_sentinel_for` from `core/constants.py`) imported by both `core/compiler.py` (filter) and `engines/accumulator/compiler.py` — so the two engines cannot diverge and the accumulator does not reach into filter-compiler internals: + +```python +def canonicalize_set_expr(col): + """Sort + dedupe a list column so equal sets compare/fingerprint identically.""" + return col.list.unique().list.sort() + +def normalize_set_expr(dim, col): + """Null → [sentinel]; canonicalize concrete lists. The single ingestion normaliser.""" + sent = unknown_sentinel_for(dim.data_type) + return ma.when(col.is_null()).then(ma.lit([sent])).otherwise(canonicalize_set_expr(col)) + +def set_wildcard_predicate(dim, col): + """True when `col` (post-normalization, non-null) is the wildcard. Detection signal.""" + return col.list.contains(ma.lit(unknown_sentinel_for(dim.data_type))) +``` + +`set_wildcard_predicate` assumes its input is already normalized (non-null); `list.contains` returns null on a null cell, so detection is always evaluated after normalization. + +**Normalization sequencing:** `normalize_set_expr` MUST run at ingestion, before any detection/coalesce/fingerprint. This is what makes every downstream predicate see only non-null lists. + +## Filter Engine (`core/compiler.py`) + +`_compile_set_membership` / `_compile_set_exclusion` wrap the existing ternary call with a wildcard short-circuit, operating on the normalized rule column: + +```python +def _compile_set_membership(self, dim): + rule_col = normalize_set_expr(dim, ma.col(dim.resolved_rule_field)) + ctx_col = ma.t_col(CTX_PREFIX + dim.dimension_name, unknown=sentinels_for(dim.data_type)) + is_wild = set_wildcard_predicate(dim, rule_col) + return ma.when(is_wild).then(0).otherwise(ctx_col.t_is_in(rule_col)) +``` + +`_compile_set_exclusion` is identical with `t_is_not_in`; a wildcard exclusion rule ("excludes nothing") is likewise ternary 0. + +- **Self-normalizing inline** — a standalone `ExpressionRulesEngine` over set rules is correct with no separate rules-frame pass. Null rule lists in input still behave as wildcards (normalized → `[sentinel]` → `is_wild` → 0): **backward-compatible**. +- **No mountainash change** — `t_is_in`'s internal `collection.is_null()` clause simply never fires post-normalization (harmless, always false). +- Ternary contract preserved: wildcard → 0, context-in-set → 1, context-out-of-set → −1, context missing (via `t_col unknown=`) → 0. + +## Accumulator Build (`engines/accumulator/`) + +The fix is in the **data**, not the join — so `_frontier_filter` is **untouched** (its inputs become null-free and canonical). + +1. **Ingestion normalization (materialized).** After `relation(rules).to_polars()`, apply `normalize_set_expr` to each set dimension's rule list column, so every `co_` starts non-null and canonical before the anchor and any coalesce. This is the step that fixes the frontier bug. +2. **Wildcard detection.** The accumulator compiler keeps its strategy-dispatching `wildcard_predicate(dim, col)` helper: the `SET_MEMBERSHIP`/`SET_EXCLUSION` branch delegates to `set_wildcard_predicate` (not `is_null()`), and the scalar/range branches keep the existing sentinel-equality test unchanged. Same set helper the filter engine uses. +3. **`_coalesce_set(dim, op)`:** + - detection via `set_wildcard_predicate`; + - both-wildcard branch → `ma.lit([sentinel])` (replaces the old `.then(co)` typed-null); + - co-wildcard → `rhs`; rhs-wildcard → `co`; + - both-concrete → `canonicalize_set_expr(set_intersection|set_union)` so equal coalesced sets share a fingerprint. + Membership uses `set_intersection`; exclusion uses `set_union`. (Wildcard is the identity for both: `∩` with match-anything, `∪` with exclude-nothing.) +4. **`_compatible_set_membership`:** `co_wild OR rhs_wild OR (intersection nonempty)` with `set_wildcard_predicate` for the wildcard tests; concrete∩concrete-nonempty unchanged. `SET_EXCLUSION` remains always-compatible (`ma.lit(True)`). +5. **`co__na` flag** = the `set_wildcard_predicate` result (emitted for provenance/apply, as today). +6. **`_frontier_filter`: unchanged.** Inputs are now null-free and canonical, so the self-join dedupes wildcard combinations, and equal-but-differently-ordered sets dedupe too. + +**Apply consistency (free).** After build, the lattice's `co_` columns *are* the normalized representation. Apply runs the filter compiler (above) over them; `set_wildcard_predicate` short-circuits identically. `normalize_set_expr` is **idempotent** — re-applying it to already-normalized `co_` columns is a no-op (the `is_null` branch never fires; canonicalizing an already-canonical list is stable) — so the filter compiler's inline normalization is safe over lattice columns. One representation, both engines, build → apply. + +## Data Flow + +``` +rule list column (may be null / unordered) + │ normalize_set_expr (ingestion — both engines) + ▼ +[sentinel] for wildcard | sorted-unique list for concrete (never null) + │ + ┌────┴───────────────────────────────┐ + ▼ filter compiler ▼ accumulator build + is_wild? → ternary 0 co_ (non-null, canonical) + else t_is_in → 1 / −1 │ coalesce (sentinel-aware) → canonical + │ co__na = is_wild + ▼ _frontier_filter (UNCHANGED) — dedupes correctly + ▼ apply → filter compiler over co_ columns (same is_wild path) +``` + +## Testing + +The old A1's fatal flaw was tests asserting only dtype / `count >= 1`. The new suite asserts **exact** results so a broken build fails loudly. + +- **Shared helpers:** `normalize_set_expr` (null→`[sentinel]`, concrete sorted-unique, dtype preserved); `canonicalize_set_expr` (`["UK","NZ"]`→`["NZ","UK"]`, dupes removed); `set_wildcard_predicate` (True on `[sentinel]`, False on concrete incl. `[]`). +- **Filter compiler:** membership/exclusion ternary — wildcard rule → **0**, context-in-set → **1**, out-of-set → **−1**, **null rule-list input → normalized → 0** (backward-compat), context-missing → 0. +- **Accumulator compiler:** coalesce both-wildcard → `[sentinel]`; wildcard-passthrough → concrete side; both-concrete → canonicalized (asserts sorted-unique) intersection (membership) / union (exclusion); `_compatible_set_membership` empty concrete∩concrete → not compatible. +- **Anchor regression (would have caught the bug):** a 3-rule all-wildcard-set build asserts `__prime_product == {30}` (NOT `{2,3,5,6,10,15,30}`); a mixed build asserts exact survivors + accumulated values. +- **Ordering:** two rules with the same set in different order → identical fingerprint → deduped to one combination. +- **Apply round-trip:** context in the `{R1,R2}` intersection matches prime-product 6; a wildcard combination matches any context — **exact** survivor counts, never `>= 1`. +- **Backend purity:** `tests/test_backend_purity.py` green; no new imports, no new `# allow:` tags. + +## Compatibility & Migration + +- **Backward-compatible input:** null rule lists are normalized at ingestion; no change to how rules are authored. Documented fast path: supply `[sentinel]` (or any concrete list) at source to skip the fill. +- **Independent of A2** (shipped, PR #51). Small `engine.py` overlap only. +- **Docs:** update `CLAUDE.md` match-strategy table — `set_membership`/`set_exclusion` are accumulator-coalesceable (membership→intersection, exclusion→union) with an **in-band `[sentinel]` wildcard** (never null); note the `null-is-not-a-portable-sentinel` principle. +- **`mountainash#89`** already covers narwhals list ops on the apply side; no new xfail group. From 7e83c37993f68627341fa289c7281daec639d8ab Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 20 Jul 2026 12:14:30 +1000 Subject: [PATCH 02/10] =?UTF-8?q?docs:=20revise=20A1=20set-wildcard=20spec?= =?UTF-8?q?=20after=20Codex=20review=20=E2=80=94=208=20findings=20addresse?= =?UTF-8?q?d?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - reject bool set dims (no typed bool sentinel) — F1 - enforce sentinel reservation + reject mixed/embedded sentinel + element-nulls at ingestion — F2/F6/F7 - narwhals scoped honestly under #89; no unified-backend claim — F3 - typed sentinel via element coercion (float), NOT native list cast (purity) — F4 - normalization a named mandatory build stage before empty/partition/anchor + pre-frontier non-null assert — F5 - co__na = wildcard predicate of FINAL coalesced value; 3-case test — F8 - cross-backend idempotence test — F9 Co-Authored-By: Claude Opus 4.8 (1M context) --- ...ccumulator-set-sentinel-wildcard-design.md | 75 ++++++++++++++----- 1 file changed, 56 insertions(+), 19 deletions(-) diff --git a/docs/superpowers/specs/2026-07-20-accumulator-set-sentinel-wildcard-design.md b/docs/superpowers/specs/2026-07-20-accumulator-set-sentinel-wildcard-design.md index 7d62509..c5b1dd5 100644 --- a/docs/superpowers/specs/2026-07-20-accumulator-set-sentinel-wildcard-design.md +++ b/docs/superpowers/specs/2026-07-20-accumulator-set-sentinel-wildcard-design.md @@ -36,16 +36,40 @@ A set-dimension **wildcard** is the single-element list `[unknown_sentinel_for(d |---|---| | str | `[""]` | | int | `[-999999999]` | -| float | `[-999999999.0]`* | +| float | `[-999999999]` cast to `List(Float64)` (see typing note) | | date | `[date(1,1,1)]` | | datetime | `[datetime(1,1,1)]` | +| **bool** | **rejected — see below** | -\* whatever `unknown_sentinel_for(float)` returns; the spec uses the existing `unknown_sentinel_for` — no new sentinels are introduced. - -The sentinel is a reserved, out-of-domain value — the same assumption the scalar `UNKNOWN` wildcard already relies on, so a concrete rule list never contains it. An **empty** concrete list `[]` keeps its concrete meaning (membership `[]` = matches nothing; exclusion `[]` = excludes nothing) and is cleanly distinct from `[sentinel]` — the empty-vs-wildcard ambiguity of the old design disappears. +An **empty** concrete list `[]` keeps its concrete meaning (membership `[]` = matches nothing; exclusion `[]` = excludes nothing) and is cleanly distinct from `[sentinel]` — the empty-vs-wildcard ambiguity of the old design disappears. Feasibility verified: `ma.lit([sentinel])` produces a `List(element)`-typed literal; `when(is_null).then(lit([sentinel])).otherwise(col)` normalizes null → `[sentinel]` and preserves dtype; `col.list.contains(lit(sentinel))` detects it. All backend-pure (route through `mountainash.expressions`), no `# allow:` tag. +### Domain constraints (enforced, not assumed) + +The wildcard sentinel is a **reserved** value, and the reservation is **validated**, not merely assumed (the scalar convention leaves it implicit; sets add a mixed-list failure mode that scalars don't have, so it must be enforced): + +1. **`bool` set dimensions are rejected.** `unknown_sentinel_for(DataType.BOOL)` falls through to the string `""` (no typed bool wildcard exists), and a set over `{true, false}` is degenerate. `Dimension` validation (`core/dimension.py`) raises `ValueError` when `match_strategy ∈ {SET_MEMBERSHIP, SET_EXCLUSION}` and `data_type == BOOL`. Fail-loud, never silently mis-typed. +2. **Concrete lists containing the sentinel are rejected at ingestion.** `normalize_set_expr` guards that a non-wildcard rule list does not contain `unknown_sentinel_for(dim.data_type)`. A list is the wildcard **iff it is exactly `[sentinel]`**; any *other* list containing the sentinel (e.g. `["AU", ""]`) is a data error and raises, so `list.contains(sentinel)` unambiguously means "wildcard". (The guard is a validation pass over the materialized rule frame at build ingestion and at filter-engine rule intake — see Failure Handling.) +3. **Element-level nulls in a concrete list are rejected at ingestion.** `[null]` / `["AU", null]` are data errors (sort/unique/membership/set-op semantics on element-nulls diverge across backends). Only the *whole-list* null (→ normalized to `[sentinel]`) is a valid wildcard. + +### Typed sentinel literal + +`unknown_sentinel_for(float)` currently returns the **int** `-999999999`. To avoid a `List(Int)` literal meeting a `List(Float)` column, the sentinel-list literal coerces its **element** to the dimension's Python type — **not** a native list-dtype cast. (A list-dtype cast is a non-goal: mountainash exposes no backend-agnostic list-dtype constructor, and casting a list literal requires a native `pl.List(...)`, which would violate backend purity — the same trap as the old null-cast. Verified: `ma.lit([-999999999.0])` with a Python-float element infers `List(Float64)` directly, and `float(-999999999) == -999999999` exactly.) + +```python +def _typed_sentinel(dim): + sent = unknown_sentinel_for(dim.data_type) + return float(sent) if dim.data_type == DataType.FLOAT else sent + +def sentinel_list_expr(dim): + """A [sentinel] list literal whose element already carries the dim's Python + type, so mountainash infers List() with no native cast (backend-pure).""" + return ma.lit([_typed_sentinel(dim)]) +``` + +All shared helpers below use `sentinel_list_expr(dim)`, never a bare `ma.lit([sent])`. + ## Shared Helpers (`core`) Three helpers live in a **new shared module `core/set_wildcard.py`** (they build `mountainash.expressions` and need `unknown_sentinel_for` from `core/constants.py`) imported by both `core/compiler.py` (filter) and `engines/accumulator/compiler.py` — so the two engines cannot diverge and the accumulator does not reach into filter-compiler internals: @@ -56,18 +80,25 @@ def canonicalize_set_expr(col): return col.list.unique().list.sort() def normalize_set_expr(dim, col): - """Null → [sentinel]; canonicalize concrete lists. The single ingestion normaliser.""" - sent = unknown_sentinel_for(dim.data_type) - return ma.when(col.is_null()).then(ma.lit([sent])).otherwise(canonicalize_set_expr(col)) + """Null → [sentinel]; canonicalize concrete lists. The single ingestion normaliser. + + Element-coerced sentinel via sentinel_list_expr(dim); no native cast. + """ + return ma.when(col.is_null()).then(sentinel_list_expr(dim)).otherwise(canonicalize_set_expr(col)) def set_wildcard_predicate(dim, col): - """True when `col` (post-normalization, non-null) is the wildcard. Detection signal.""" - return col.list.contains(ma.lit(unknown_sentinel_for(dim.data_type))) + """True when `col` (post-normalization + validation) is the wildcard. + + Because ingestion validation rejects any concrete list containing the + sentinel, `list.contains(sentinel)` is true iff the list is exactly + [sentinel]. Evaluated only after normalization (contains() is null on a + null cell).""" + return col.list.contains(ma.lit(_typed_sentinel(dim))) # typed scalar sentinel ``` -`set_wildcard_predicate` assumes its input is already normalized (non-null); `list.contains` returns null on a null cell, so detection is always evaluated after normalization. +**Ingestion validation (guards the reservation, F2/F6/F7).** A separate validation pass over the materialized rule frame — run once at ingestion in *both* engines — raises `ValueError` when, for any set dimension, a **non-null** rule list either (a) contains the sentinel but is not exactly `[sentinel]` (mixed/embedded sentinel), or (b) contains a null element. This makes "reserved, out-of-domain" an enforced contract, not an assumption, so `set_wildcard_predicate` is unambiguous. Whole-list null is *not* an error — it is the valid wildcard, normalized to `[sentinel]`. -**Normalization sequencing:** `normalize_set_expr` MUST run at ingestion, before any detection/coalesce/fingerprint. This is what makes every downstream predicate see only non-null lists. +**Normalization sequencing:** validation → `normalize_set_expr` MUST run at ingestion, before any detection/coalesce/fingerprint. This is what makes every downstream predicate see only non-null, validated lists. ## Filter Engine (`core/compiler.py`) @@ -86,21 +117,22 @@ def _compile_set_membership(self, dim): - **Self-normalizing inline** — a standalone `ExpressionRulesEngine` over set rules is correct with no separate rules-frame pass. Null rule lists in input still behave as wildcards (normalized → `[sentinel]` → `is_wild` → 0): **backward-compatible**. - **No mountainash change** — `t_is_in`'s internal `collection.is_null()` clause simply never fires post-normalization (harmless, always false). - Ternary contract preserved: wildcard → 0, context-in-set → 1, context-out-of-set → −1, context missing (via `t_col unknown=`) → 0. +- **Backend scope, stated honestly (F3).** This does **not** widen backend support. `t_is_in`/`list.contains` on a list *column* still hits the documented Narwhals limitation (Narwhals rejects an expression argument to `list.contains`), the same gap already tracked by `mountainash#89`. The accumulator build is polars-internal so build is unaffected; the filter/apply path over set list columns remains **polars/ibis only**, with Narwhals under the existing `#89` xfail — no new xfail group, and the spec makes **no unified-all-backends claim** for set list evaluation. ## Accumulator Build (`engines/accumulator/`) -The fix is in the **data**, not the join — so `_frontier_filter` is **untouched** (its inputs become null-free and canonical). +The fix is in the **data**, not the join — so `_frontier_filter` is **untouched** (its inputs become null-free and canonical). The safety of "no change" depends entirely on normalization reaching *every* anchor path, so it is a named, mandatory stage: -1. **Ingestion normalization (materialized).** After `relation(rules).to_polars()`, apply `normalize_set_expr` to each set dimension's rule list column, so every `co_` starts non-null and canonical before the anchor and any coalesce. This is the step that fixes the frontier bug. +1. **`_normalize_set_columns(rules_pl)` — a named build stage (F5).** Immediately after `relation(rules).to_polars()` and **before** the empty-frame branch, the partition output, and `_create_anchor` — for *every* build path (empty input, partition-filtered input, and any imported/flat lattice path that reaches the anchor) — run ingestion validation (reservation + null-element guards) then apply `normalize_set_expr` to each set dimension's rule list column. Every `co_` for a set dim is therefore non-null and canonical before the anchor and any coalesce. A build-time assertion confirms no set `co_` column is null before `_frontier_filter`. This stage is what makes the frontier "no change" claim sound. 2. **Wildcard detection.** The accumulator compiler keeps its strategy-dispatching `wildcard_predicate(dim, col)` helper: the `SET_MEMBERSHIP`/`SET_EXCLUSION` branch delegates to `set_wildcard_predicate` (not `is_null()`), and the scalar/range branches keep the existing sentinel-equality test unchanged. Same set helper the filter engine uses. 3. **`_coalesce_set(dim, op)`:** - detection via `set_wildcard_predicate`; - - both-wildcard branch → `ma.lit([sentinel])` (replaces the old `.then(co)` typed-null); + - both-wildcard branch → `sentinel_list_expr(dim)` (typed `[sentinel]` literal; replaces the old `.then(co)` typed-null); - co-wildcard → `rhs`; rhs-wildcard → `co`; - both-concrete → `canonicalize_set_expr(set_intersection|set_union)` so equal coalesced sets share a fingerprint. Membership uses `set_intersection`; exclusion uses `set_union`. (Wildcard is the identity for both: `∩` with match-anything, `∪` with exclude-nothing.) 4. **`_compatible_set_membership`:** `co_wild OR rhs_wild OR (intersection nonempty)` with `set_wildcard_predicate` for the wildcard tests; concrete∩concrete-nonempty unchanged. `SET_EXCLUSION` remains always-compatible (`ma.lit(True)`). -5. **`co__na` flag** = the `set_wildcard_predicate` result (emitted for provenance/apply, as today). +5. **`co__na` flag = `set_wildcard_predicate` applied to the FINAL coalesced value (F8)**, not to the LHS/RHS inputs. So wildcard+concrete → concrete coalesced value → flag **false**; wildcard+wildcard → `[sentinel]` → flag **true**; concrete+concrete → concrete → false. The flag tracks whether the *combination* leaves the dimension unconstrained, which is the semantics the frontier fingerprint and apply/provenance need. 6. **`_frontier_filter`: unchanged.** Inputs are now null-free and canonical, so the self-join dedupes wildcard combinations, and equal-but-differently-ordered sets dedupe too. **Apply consistency (free).** After build, the lattice's `co_` columns *are* the normalized representation. Apply runs the filter compiler (above) over them; `set_wildcard_predicate` short-circuits identically. `normalize_set_expr` is **idempotent** — re-applying it to already-normalized `co_` columns is a no-op (the `is_null` branch never fires; canonicalizing an already-canonical list is stable) — so the filter compiler's inline normalization is safe over lattice columns. One representation, both engines, build → apply. @@ -126,17 +158,22 @@ rule list column (may be null / unordered) The old A1's fatal flaw was tests asserting only dtype / `count >= 1`. The new suite asserts **exact** results so a broken build fails loudly. -- **Shared helpers:** `normalize_set_expr` (null→`[sentinel]`, concrete sorted-unique, dtype preserved); `canonicalize_set_expr` (`["UK","NZ"]`→`["NZ","UK"]`, dupes removed); `set_wildcard_predicate` (True on `[sentinel]`, False on concrete incl. `[]`). +- **Shared helpers:** `sentinel_list_expr` (element carries dim Python type; float dim → `List(Float64)` literal, verified); `normalize_set_expr` (null→`[sentinel]`, concrete sorted-unique, dtype preserved); `canonicalize_set_expr` (`["UK","NZ"]`→`["NZ","UK"]`, dupes removed); `set_wildcard_predicate` (True on `[sentinel]`, False on concrete incl. `[]`). +- **Domain validation (F1/F2/F6/F7):** `Dimension(SET_MEMBERSHIP|SET_EXCLUSION, data_type=bool)` raises `ValueError`; ingestion rejects a concrete list containing the sentinel (`["AU",""]` → raise) and rejects element-level nulls (`["AU", null]` → raise); whole-list null is accepted (→ wildcard). +- **Float typing (F4):** a `float` set dimension normalizes and coalesces to `List(Float64)` (element `-999999999.0`), asserted on polars; the `sentinel_list_expr` element-type is checked per dtype (str/int/float/date/datetime). - **Filter compiler:** membership/exclusion ternary — wildcard rule → **0**, context-in-set → **1**, out-of-set → **−1**, **null rule-list input → normalized → 0** (backward-compat), context-missing → 0. - **Accumulator compiler:** coalesce both-wildcard → `[sentinel]`; wildcard-passthrough → concrete side; both-concrete → canonicalized (asserts sorted-unique) intersection (membership) / union (exclusion); `_compatible_set_membership` empty concrete∩concrete → not compatible. -- **Anchor regression (would have caught the bug):** a 3-rule all-wildcard-set build asserts `__prime_product == {30}` (NOT `{2,3,5,6,10,15,30}`); a mixed build asserts exact survivors + accumulated values. +- **`co__na` post-coalesce (F8):** assert the flag on the final coalesced value for all three cases — wildcard+wildcard → **1**, wildcard+concrete → **0**, concrete+concrete → **0**. +- **Anchor regression (would have caught the bug):** a 3-rule all-wildcard-set build asserts `__prime_product == {30}` (NOT `{2,3,5,6,10,15,30}`); a mixed build asserts exact survivors + accumulated values. Plus a **pre-frontier assertion** that no set `co_` column is null on every path (empty, partitioned, multi-level). - **Ordering:** two rules with the same set in different order → identical fingerprint → deduped to one combination. - **Apply round-trip:** context in the `{R1,R2}` intersection matches prime-product 6; a wildcard combination matches any context — **exact** survivor counts, never `>= 1`. +- **Idempotence (F9):** `normalize(normalize(x))` equals `normalize(x)` at collected-value level, across dtypes (float/date included) on the polars build path. - **Backend purity:** `tests/test_backend_purity.py` green; no new imports, no new `# allow:` tags. ## Compatibility & Migration - **Backward-compatible input:** null rule lists are normalized at ingestion; no change to how rules are authored. Documented fast path: supply `[sentinel]` (or any concrete list) at source to skip the fill. +- **New validations (fail-loud, not silent):** `bool` set dimensions, sentinel-in-concrete-list, and element-level nulls now raise `ValueError`. These reject only inputs that were already degenerate or ambiguous — no previously-correct rule set is affected (bool set dims and sentinel-bearing lists had no well-defined meaning before). - **Independent of A2** (shipped, PR #51). Small `engine.py` overlap only. -- **Docs:** update `CLAUDE.md` match-strategy table — `set_membership`/`set_exclusion` are accumulator-coalesceable (membership→intersection, exclusion→union) with an **in-band `[sentinel]` wildcard** (never null); note the `null-is-not-a-portable-sentinel` principle. -- **`mountainash#89`** already covers narwhals list ops on the apply side; no new xfail group. +- **Docs:** update `CLAUDE.md` match-strategy table — `set_membership`/`set_exclusion` are accumulator-coalesceable (membership→intersection, exclusion→union) with an **in-band `[sentinel]` wildcard** (never null; `bool` unsupported); note the `null-is-not-a-portable-sentinel` principle. +- **`mountainash#89`** already covers narwhals list ops on the apply side; no new xfail group; no unified-all-backends claim for set list evaluation. From 08038d879d565ee29ab5f5dc6bc4057f367fd367 Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 20 Jul 2026 12:24:14 +1000 Subject: [PATCH 03/10] docs: A1 set-wildcard in-band sentinel implementation plan (6 tasks) 6 TDD tasks: bool-set rejection; core/set_wildcard.py helpers+validation; filter compiler short-circuit+engine validation; accumulator compiler set branches; engine normalization stage+seed NA+pre-frontier assertion; apply round-trip+float+docs. Frontier untouched; backend-pure throughout. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...07-20-accumulator-set-sentinel-wildcard.md | 1103 +++++++++++++++++ 1 file changed, 1103 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-20-accumulator-set-sentinel-wildcard.md diff --git a/docs/superpowers/plans/2026-07-20-accumulator-set-sentinel-wildcard.md b/docs/superpowers/plans/2026-07-20-accumulator-set-sentinel-wildcard.md new file mode 100644 index 0000000..a9cd05f --- /dev/null +++ b/docs/superpowers/plans/2026-07-20-accumulator-set-sentinel-wildcard.md @@ -0,0 +1,1103 @@ +# Accumulator Set-Wildcard In-Band Sentinel — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Represent a set-dimension wildcard as the in-band list `[unknown_sentinel_for(dtype)]` (never null) across the filter engine and the accumulator build, so `_frontier_filter` dedupes correctly and `apply` returns correct survivors/aggregates. + +**Architecture:** A new shared module `core/set_wildcard.py` holds the sentinel representation (construct / normalize / detect / validate). The filter compiler short-circuits set ternaries on the wildcard; the accumulator normalizes set columns at a named ingestion stage so `_frontier_filter` needs no change. Bool set dims, sentinel-embedding lists, and element-nulls are rejected fail-loud. + +**Tech Stack:** Python 3.10+, `mountainash.expressions` / `mountainash.relations` (backend-agnostic), pydantic, polars (build materialisation), pytest. + +**Design spec:** `docs/superpowers/specs/2026-07-20-accumulator-set-sentinel-wildcard-design.md` + +## Global Constraints + +- **Backend purity (ENFORCED).** No module under `src/mountainash_rules/` may import polars/ibis/narwhals except the three existing `# allow:`-tagged lines. This work adds **zero** new native imports and **zero** new `# allow:` tags — `ma.lit([...])`, `list.contains`, `list.unique`, `list.sort`, `list.len`, `list.drop_nulls`, `list.set_intersection`, `list.set_union` route through `mountainash.expressions`; `validate_set_columns` uses `relation(...).filter(...).collect()` + builtin `len()`. `tests/test_backend_purity.py` must stay green. +- **Wildcard = exactly `[unknown_sentinel_for(dim.data_type)]`**, element coerced to the dim's Python type (float sentinel → `-999999999.0`). Never a bare `ma.lit([sent])` — always `sentinel_list_expr(dim)`. Never a native list-dtype cast (no `pl.List(...)` — purity + it is refused). +- **Null-list = wildcard** (normalized to `[sentinel]`); **empty list `[]`** keeps concrete meaning (membership = matches nothing); **bool set dims / sentinel-embedding lists / element-nulls** are rejected with `ValueError`. +- **`_frontier_filter`, `_check_overflow`, partition routing, scalar/range/string strategies: DO NOT TOUCH.** The fix is normalization at ingestion, not the join. +- **Build is polars-internal** (`engine.py:121` `to_polars()`); set-ops/normalization run on polars. No new backend xfails; the only backend gap is the pre-existing apply-phase `mountainash#89` (narwhals list ops). +- **TDD**: failing test first, one concern at a time. `ValueError` for validation. +- Fast suite: `hatch run test:test-quick`; single: `hatch run test:test-target `; lint: `hatch run ruff:check `. + +--- + +### Task 1: Reject `bool` set dimensions (F1) + +**Files:** +- Modify: `src/mountainash_rules/core/dimension.py` (the `_validate_strategy_fields` model validator) +- Test: `tests/core/test_dimension.py` (new test class) + +**Interfaces:** +- Produces: `Dimension(match_strategy=SET_MEMBERSHIP|SET_EXCLUSION, data_type=bool)` raises `ValueError`. +- Consumes: nothing. + +- [ ] **Step 1: Write the failing test** + +Add to `tests/core/test_dimension.py`: +```python +class TestSetDimensionBoolRejected: + def test_bool_set_membership_rejected(self): + import pytest + from pydantic import ValidationError + from mountainash_rules import Dimension + from mountainash_rules.core.constants import MatchStrategy, DataType + with pytest.raises(ValidationError, match="bool"): + Dimension(dimension_name="flags", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.BOOL) + + def test_bool_set_exclusion_rejected(self): + import pytest + from pydantic import ValidationError + from mountainash_rules import Dimension + from mountainash_rules.core.constants import MatchStrategy, DataType + with pytest.raises(ValidationError, match="bool"): + Dimension(dimension_name="flags", match_strategy=MatchStrategy.SET_EXCLUSION, data_type=DataType.BOOL) + + def test_str_set_membership_allowed(self): + from mountainash_rules import Dimension + from mountainash_rules.core.constants import MatchStrategy, DataType + d = Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.STR) + assert d.data_type is DataType.STR + + def test_int_set_membership_allowed(self): + from mountainash_rules import Dimension + from mountainash_rules.core.constants import MatchStrategy, DataType + d = Dimension(dimension_name="tiers", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.INT) + assert d.data_type is DataType.INT +``` + +- [ ] **Step 2: Run it to verify it fails** + +Run: `hatch run test:test-target tests/core/test_dimension.py::TestSetDimensionBoolRejected -v` +Expected: the two `_rejected` tests FAIL (no error raised); the two `_allowed` tests PASS. + +- [ ] **Step 3: Add the rejection to `_validate_strategy_fields`** + +In `src/mountainash_rules/core/dimension.py`, inside `_validate_strategy_fields`, immediately before the final `return self`, add: +```python + if self.match_strategy in ( + MatchStrategy.SET_MEMBERSHIP, + MatchStrategy.SET_EXCLUSION, + ): + if self.data_type is DataType.BOOL: + raise ValueError( + f"Dimension '{self.dimension_name}' uses " + f"{self.match_strategy.value} with data_type bool; boolean " + f"set dimensions are not supported (no typed wildcard sentinel " + f"exists and a set over {{true, false}} is degenerate)" + ) +``` + +- [ ] **Step 4: Run the test to verify it passes** + +Run: `hatch run test:test-target tests/core/test_dimension.py::TestSetDimensionBoolRejected -v` +Expected: PASS (4 tests). + +- [ ] **Step 5: Run the core dimension suite for regressions** + +Run: `hatch run test:test-target tests/core/test_dimension.py -q` +Expected: PASS (no existing test regressed). + +- [ ] **Step 6: Commit** + +```bash +git add src/mountainash_rules/core/dimension.py tests/core/test_dimension.py +git commit -m "feat(dimension): reject bool set dimensions (no typed wildcard sentinel)" +``` + +--- + +### Task 2: Shared `core/set_wildcard.py` module (F2, F4, F6, F7) + +**Files:** +- Create: `src/mountainash_rules/core/set_wildcard.py` +- Test: `tests/core/test_set_wildcard.py` (new file) + +**Interfaces:** +- Produces (all imported by Tasks 3–5): + - `sentinel_list_expr(dim: Dimension) -> BaseExpressionAPI` — a `[sentinel]` list literal, element typed to the dim. + - `canonicalize_set_expr(col: BaseExpressionAPI) -> BaseExpressionAPI` — sort + unique. + - `normalize_set_expr(dim, col) -> BaseExpressionAPI` — null→`[sentinel]`, concrete→sorted-unique. + - `set_wildcard_predicate(dim, col) -> BaseExpressionAPI` — `col.list.contains(sentinel)` (post-normalization). + - `validate_set_columns(rules_rel, set_dims: list[Dimension]) -> None` — raises `ValueError` on embedded-sentinel or element-null rule lists. +- Consumes: `unknown_sentinel_for`, `DataType` from `core/constants.py`; `Dimension` from `core/dimension.py`. + +- [ ] **Step 1: Write the failing helper tests** + +Create `tests/core/test_set_wildcard.py`: +```python +"""Tests for the shared set-wildcard sentinel helpers.""" + +import polars as pl +import pytest + +from mountainash_rules.core.constants import MatchStrategy, DataType +from mountainash_rules.core.dimension import Dimension +from mountainash_rules.core.set_wildcard import ( + sentinel_list_expr, + canonicalize_set_expr, + normalize_set_expr, + set_wildcard_predicate, + validate_set_columns, +) +from mountainash.relations import relation + + +def _str_dim(): + return Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.STR) + + +def _float_dim(): + return Dimension(dimension_name="scores", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.FLOAT) + + +class TestNormalizeAndDetect: + def test_null_becomes_sentinel_list(self): + dim = _str_dim() + df = pl.DataFrame({"region": pl.Series("region", [None, ["AU"]], dtype=pl.List(pl.Utf8))}) + out = df.with_columns(normalize_set_expr(dim, __import__("mountainash").col("region")).alias("n").compile(df, booleanizer=None)) + assert out["n"].to_list() == [[""], ["AU"]] + assert out["n"].dtype == pl.List(pl.Utf8) + + def test_concrete_list_sorted_and_deduped(self): + dim = _str_dim() + import mountainash as ma + df = pl.DataFrame({"region": pl.Series("region", [["UK", "NZ", "NZ"]], dtype=pl.List(pl.Utf8))}) + out = df.with_columns(normalize_set_expr(dim, ma.col("region")).alias("n").compile(df, booleanizer=None)) + assert out["n"].to_list() == [["NZ", "UK"]] + + def test_wildcard_predicate_true_on_sentinel_false_on_concrete(self): + dim = _str_dim() + import mountainash as ma + df = pl.DataFrame({"region": pl.Series("region", [[""], ["AU"], []], dtype=pl.List(pl.Utf8))}) + out = df.with_columns(set_wildcard_predicate(dim, ma.col("region")).alias("w").compile(df, booleanizer=None)) + assert out["w"].to_list() == [True, False, False] + + def test_float_dim_sentinel_list_is_float_typed(self): + dim = _float_dim() + import mountainash as ma + df = pl.DataFrame({"scores": pl.Series("scores", [None], dtype=pl.List(pl.Float64))}) + out = df.with_columns(normalize_set_expr(dim, ma.col("scores")).alias("n").compile(df, booleanizer=None)) + assert out["n"].dtype == pl.List(pl.Float64) + assert out["n"].to_list() == [[-999999999.0]] + + def test_normalize_idempotent(self): + dim = _str_dim() + import mountainash as ma + df = pl.DataFrame({"region": pl.Series("region", [None, ["UK", "NZ"]], dtype=pl.List(pl.Utf8))}) + once = df.with_columns(normalize_set_expr(dim, ma.col("region")).alias("region").compile(df, booleanizer=None)) + twice = once.with_columns(normalize_set_expr(dim, ma.col("region")).alias("region").compile(once, booleanizer=None)) + assert once["region"].to_list() == twice["region"].to_list() + + +class TestCanonicalize: + def test_sort_and_dedupe(self): + import mountainash as ma + df = pl.DataFrame({"c": pl.Series("c", [["UK", "NZ", "UK"]], dtype=pl.List(pl.Utf8))}) + out = df.with_columns(canonicalize_set_expr(ma.col("c")).alias("c2").compile(df, booleanizer=None)) + assert out["c2"].to_list() == [["NZ", "UK"]] + + +class TestValidateSetColumns: + def test_embedded_sentinel_rejected(self): + dim = _str_dim() + rules = pl.DataFrame({"region": pl.Series("region", [["AU", ""]], dtype=pl.List(pl.Utf8))}) + with pytest.raises(ValueError, match="sentinel"): + validate_set_columns(relation(rules), [dim]) + + def test_null_element_rejected(self): + dim = _str_dim() + rules = pl.DataFrame({"region": pl.Series("region", [["AU", None]], dtype=pl.List(pl.Utf8))}) + with pytest.raises(ValueError, match="null element"): + validate_set_columns(relation(rules), [dim]) + + def test_whole_list_null_is_allowed(self): + dim = _str_dim() + rules = pl.DataFrame({"region": pl.Series("region", [None, ["AU"], [""]], dtype=pl.List(pl.Utf8))}) + validate_set_columns(relation(rules), [dim]) # no raise + + def test_no_set_dims_is_noop(self): + validate_set_columns(relation(pl.DataFrame({"x": [1]})), []) +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `hatch run test:test-target tests/core/test_set_wildcard.py -v` +Expected: FAIL — `ModuleNotFoundError: No module named 'mountainash_rules.core.set_wildcard'`. + +- [ ] **Step 3: Create the module** + +Create `src/mountainash_rules/core/set_wildcard.py`: +```python +"""Shared in-band sentinel representation for set-dimension wildcards. + +A SET_MEMBERSHIP/SET_EXCLUSION wildcard is the single-element list +``[unknown_sentinel_for(dim.data_type)]`` — never a null. These helpers are the +single source of truth for both the filter engine (``core/compiler.py``) and the +accumulator engine (``engines/accumulator/``), so the two cannot diverge. + +The sentinel is a reserved, out-of-domain value; ``validate_set_columns`` +enforces that a concrete rule list never embeds it, so ``set_wildcard_predicate`` +is unambiguous. +""" + +from __future__ import annotations + +import typing as t + +import mountainash.expressions as ma +from mountainash.expressions import BaseExpressionAPI + +from mountainash_rules.core.constants import DataType, unknown_sentinel_for +from mountainash_rules.core.dimension import Dimension + + +def _typed_sentinel(dim: Dimension) -> t.Any: + """The dimension's wildcard sentinel, coerced to the dim's Python element type.""" + sent = unknown_sentinel_for(dim.data_type) + return float(sent) if dim.data_type == DataType.FLOAT else sent + + +def sentinel_list_expr(dim: Dimension) -> BaseExpressionAPI: + """A ``[sentinel]`` list literal whose element carries the dim's Python type. + + Element coercion (not a native list-dtype cast) keeps this backend-pure: a + Python-float element yields a ``List(Float64)`` literal with no cast. + """ + return ma.lit([_typed_sentinel(dim)]) + + +def canonicalize_set_expr(col: BaseExpressionAPI) -> BaseExpressionAPI: + """Sort + dedupe a list column so equal sets compare/fingerprint identically.""" + return col.list.unique().list.sort() + + +def normalize_set_expr(dim: Dimension, col: BaseExpressionAPI) -> BaseExpressionAPI: + """Null list -> ``[sentinel]``; concrete list -> sorted-unique. The one normaliser. + + Idempotent: re-applying to an already-normalized column is a no-op. + """ + return ma.when(col.is_null()).then(sentinel_list_expr(dim)).otherwise( + canonicalize_set_expr(col) + ) + + +def set_wildcard_predicate(dim: Dimension, col: BaseExpressionAPI) -> BaseExpressionAPI: + """True when ``col`` (post-normalization) is the wildcard. + + Because ``validate_set_columns`` rejects any concrete list embedding the + sentinel, ``list.contains(sentinel)`` is true iff the list is exactly + ``[sentinel]``. Must be evaluated after normalization (``contains`` returns + null on a null cell). + """ + return col.list.contains(ma.lit(_typed_sentinel(dim))) + + +def _embedded_sentinel_predicate(dim: Dimension, field: str) -> BaseExpressionAPI: + """True for a non-null list that contains the sentinel but is not ``[sentinel]``.""" + col = ma.col(field) + has_sentinel = col.list.contains(ma.lit(_typed_sentinel(dim))) + length = col.list.len() + return col.is_not_null().__and__(has_sentinel.__and__(length.ne(ma.lit(1)))) + + +def _null_element_predicate(field: str) -> BaseExpressionAPI: + """True for a non-null list that contains a null element.""" + col = ma.col(field) + length = col.list.len() + non_null_length = col.list.drop_nulls().list.len() + return col.is_not_null().__and__(length.ne(non_null_length)) + + +def validate_set_columns(rules_rel: t.Any, set_dims: list[Dimension]) -> None: + """Raise ``ValueError`` if any set-dimension rule list is a data error. + + A data error is (a) a concrete list embedding the sentinel among other + values, or (b) a list with a null element. A whole-list null is valid — it is + the wildcard. ``rules_rel`` is a ``mountainash`` relation; the check collects + the offending rows and uses builtin ``len`` (backend-pure — ``collect`` + returns a native frame that supports ``len``). + """ + for dim in set_dims: + field = dim.resolved_rule_field + embedded = rules_rel.filter(_embedded_sentinel_predicate(dim, field)).collect() + if len(embedded) > 0: + raise ValueError( + f"Dimension '{dim.dimension_name}': a concrete rule list embeds the " + f"reserved wildcard sentinel {unknown_sentinel_for(dim.data_type)!r}. " + f"The sentinel is only valid as the sole element (the wildcard)." + ) + null_elem = rules_rel.filter(_null_element_predicate(field)).collect() + if len(null_elem) > 0: + raise ValueError( + f"Dimension '{dim.dimension_name}': a rule list contains a null " + f"element. Element-level nulls are not allowed; use a whole-list " + f"null (or omit the cell) for a wildcard." + ) +``` + +- [ ] **Step 4: Run the helper tests to verify they pass** + +Run: `hatch run test:test-target tests/core/test_set_wildcard.py -v` +Expected: PASS (all tests). + +- [ ] **Step 5: Run backend purity** + +Run: `hatch run test:test-target tests/test_backend_purity.py -q` +Expected: PASS — the new module imports only `mountainash.expressions` (no polars/ibis/narwhals). + +- [ ] **Step 6: Commit** + +```bash +git add src/mountainash_rules/core/set_wildcard.py tests/core/test_set_wildcard.py +git commit -m "feat(core): shared in-band sentinel set-wildcard helpers + validation" +``` + +--- + +### Task 3: Filter compiler + filter-engine validation (F3) + +**Files:** +- Modify: `src/mountainash_rules/core/compiler.py` (`_compile_set_membership`, `_compile_set_exclusion`) +- Modify: `src/mountainash_rules/engines/filter/engine.py` (validate set columns once) +- Test: `tests/core/test_compiler.py` (filter compiler ternary tests) + +**Interfaces:** +- Consumes: `normalize_set_expr`, `set_wildcard_predicate`, `validate_set_columns` (Task 2). +- Produces: filter set ternaries that short-circuit the wildcard to 0 and normalize null input; the filter engine rejects invalid set rule lists. + +- [ ] **Step 1: Write the failing filter compiler tests** + +Add to `tests/core/test_compiler.py` (top-of-file imports already include `pl`, `DimensionCompiler`, `Dimension`, `MatchStrategy`; add `from mountainash_rules.core.constants import CTX_PREFIX` if not present): +```python +class TestSetMembershipTernary: + def _compile(self, dim): + from mountainash_rules.core.compiler import DimensionCompiler + return DimensionCompiler().compile_dimension(dim) + + def _dim(self): + from mountainash_rules.core.constants import DataType + return Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.STR) + + def test_wildcard_rule_is_ternary_zero(self): + from mountainash_rules.core.constants import CTX_PREFIX + expr = self._compile(self._dim()) + df = pl.DataFrame({ + "region": pl.Series("region", [[""]], dtype=pl.List(pl.Utf8)), + CTX_PREFIX + "region": ["AU"], + }) + out = df.with_columns(expr.alias("t").compile(df, booleanizer=None)) + assert out["t"].to_list() == [0] + + def test_context_in_set_is_one(self): + from mountainash_rules.core.constants import CTX_PREFIX + expr = self._compile(self._dim()) + df = pl.DataFrame({ + "region": pl.Series("region", [["AU", "NZ"]], dtype=pl.List(pl.Utf8)), + CTX_PREFIX + "region": ["AU"], + }) + out = df.with_columns(expr.alias("t").compile(df, booleanizer=None)) + assert out["t"].to_list() == [1] + + def test_context_out_of_set_is_minus_one(self): + from mountainash_rules.core.constants import CTX_PREFIX + expr = self._compile(self._dim()) + df = pl.DataFrame({ + "region": pl.Series("region", [["AU", "NZ"]], dtype=pl.List(pl.Utf8)), + CTX_PREFIX + "region": ["US"], + }) + out = df.with_columns(expr.alias("t").compile(df, booleanizer=None)) + assert out["t"].to_list() == [-1] + + def test_null_rule_list_normalizes_to_wildcard(self): + from mountainash_rules.core.constants import CTX_PREFIX + expr = self._compile(self._dim()) + df = pl.DataFrame({ + "region": pl.Series("region", [None], dtype=pl.List(pl.Utf8)), + CTX_PREFIX + "region": ["AU"], + }) + out = df.with_columns(expr.alias("t").compile(df, booleanizer=None)) + assert out["t"].to_list() == [0] + + +class TestSetExclusionTernary: + def _compile(self, dim): + from mountainash_rules.core.compiler import DimensionCompiler + return DimensionCompiler().compile_dimension(dim) + + def _dim(self): + from mountainash_rules.core.constants import DataType + return Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_EXCLUSION, data_type=DataType.STR) + + def test_wildcard_rule_is_zero(self): + from mountainash_rules.core.constants import CTX_PREFIX + expr = self._compile(self._dim()) + df = pl.DataFrame({ + "region": pl.Series("region", [[""]], dtype=pl.List(pl.Utf8)), + CTX_PREFIX + "region": ["AU"], + }) + out = df.with_columns(expr.alias("t").compile(df, booleanizer=None)) + assert out["t"].to_list() == [0] + + def test_context_in_excluded_set_is_minus_one(self): + from mountainash_rules.core.constants import CTX_PREFIX + expr = self._compile(self._dim()) + df = pl.DataFrame({ + "region": pl.Series("region", [["AU"]], dtype=pl.List(pl.Utf8)), + CTX_PREFIX + "region": ["AU"], + }) + out = df.with_columns(expr.alias("t").compile(df, booleanizer=None)) + assert out["t"].to_list() == [-1] + + def test_context_not_in_excluded_set_is_one(self): + from mountainash_rules.core.constants import CTX_PREFIX + expr = self._compile(self._dim()) + df = pl.DataFrame({ + "region": pl.Series("region", [["AU"]], dtype=pl.List(pl.Utf8)), + CTX_PREFIX + "region": ["NZ"], + }) + out = df.with_columns(expr.alias("t").compile(df, booleanizer=None)) + assert out["t"].to_list() == [1] +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `hatch run test:test-target tests/core/test_compiler.py::TestSetMembershipTernary tests/core/test_compiler.py::TestSetExclusionTernary -v` +Expected: `test_wildcard_rule_is_ternary_zero` / `test_wildcard_rule_is_zero` FAIL — current code returns `-1` for a `[""]` rule (no short-circuit); `test_null_rule_list_normalizes_to_wildcard` passes today via `t_is_in`'s null handling but must keep passing after the change. + +- [ ] **Step 3: Rewrite the two filter set methods** + +In `src/mountainash_rules/core/compiler.py`, add to the imports block (next to the other `mountainash_rules.core` imports): +```python +from mountainash_rules.core.set_wildcard import normalize_set_expr, set_wildcard_predicate +``` +Replace `_compile_set_membership` and `_compile_set_exclusion` with: +```python + def _compile_set_membership(self, dim: Dimension) -> BaseExpressionAPI: + """SET_MEMBERSHIP: context value in the rule list; wildcard rule -> ternary 0.""" + rule_col = normalize_set_expr(dim, ma.col(dim.resolved_rule_field)) + ctx_col = ma.t_col(CTX_PREFIX + dim.dimension_name, unknown=sentinels_for(dim.data_type)) + is_wild = set_wildcard_predicate(dim, rule_col) + return ma.when(is_wild).then(0).otherwise(ctx_col.t_is_in(rule_col)) + + def _compile_set_exclusion(self, dim: Dimension) -> BaseExpressionAPI: + """SET_EXCLUSION: context value NOT in the rule list; wildcard rule -> ternary 0.""" + rule_col = normalize_set_expr(dim, ma.col(dim.resolved_rule_field)) + ctx_col = ma.t_col(CTX_PREFIX + dim.dimension_name, unknown=sentinels_for(dim.data_type)) + is_wild = set_wildcard_predicate(dim, rule_col) + return ma.when(is_wild).then(0).otherwise(ctx_col.t_is_not_in(rule_col)) +``` + +- [ ] **Step 4: Run the compiler tests to verify they pass** + +Run: `hatch run test:test-target tests/core/test_compiler.py::TestSetMembershipTernary tests/core/test_compiler.py::TestSetExclusionTernary -v` +Expected: PASS. + +- [ ] **Step 5: Write the failing filter-engine validation test** + +Add to `tests/core/test_compiler.py` (or wherever the filter engine is tested — use `tests/test_engine.py` if that is the filter-engine suite; this test uses the public `ExpressionRulesEngine`): +```python +class TestFilterEngineRejectsInvalidSetRules: + def test_embedded_sentinel_rule_rejected_on_evaluate(self): + import polars as pl + import pytest + from mountainash_rules import ExpressionRulesEngine, Dimension, DimensionsMetadata + from mountainash_rules.core.constants import MatchStrategy, DataType + meta = DimensionsMetadata(dimensions=[ + Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.STR), + ]) + rules = pl.DataFrame({ + "rule_name": ["R1"], + "region": pl.Series("region", [["AU", ""]], dtype=pl.List(pl.Utf8)), + }) + engine = ExpressionRulesEngine(rules=rules, dimension_metadata=meta) + with pytest.raises(ValueError, match="sentinel"): + engine.evaluate({"region": "AU"}) +``` +> Verified: `ExpressionRulesEngine(rules=..., dimension_metadata=...)` (`engine.py:55`); `evaluate` accepts a dict. Both `ExpressionRulesEngine` and `Dimension`/`DimensionsMetadata` are public root imports. + +- [ ] **Step 6: Run to verify it fails** + +Run: `hatch run test:test-target tests/core/test_compiler.py::TestFilterEngineRejectsInvalidSetRules -v` +Expected: FAIL — no `ValueError` raised (invalid list silently treated as wildcard). + +- [ ] **Step 7: Add memoized validation to the filter engine** + +In `src/mountainash_rules/engines/filter/engine.py`, add the import near the other `mountainash_rules` imports: +```python +from mountainash_rules.core.set_wildcard import validate_set_columns +from mountainash_rules.core.constants import MatchStrategy +``` +In `_scored_relation` (around line 407–412), immediately after it builds the rules relation (`rel = relation(self._rules)`), add a one-time validation guarded by a flag. First, in `__init__` (near `self._rules = rules`, line 74), add: +```python + self._set_dims_validated = False +``` +Then in `_scored_relation`, right after `rel = relation(self._rules)`: +```python + if self._metadata is not None and not self._set_dims_validated: + set_dims = [ + d for d in self._metadata.dimensions + if d.match_strategy in (MatchStrategy.SET_MEMBERSHIP, MatchStrategy.SET_EXCLUSION) + ] + validate_set_columns(rel, set_dims) + self._set_dims_validated = True +``` +> Verified: metadata attribute is `self._metadata` (`engine.py:69`), and it is **`None`** when the engine was constructed from `dimension_expressions` instead of metadata — hence the `self._metadata is not None` guard (the expressions path has no `Dimension` objects to inspect). `_scored_relation` is shared by `evaluate`, `explain`, and `evaluate_batch`, so validating here covers all entry points. + +- [ ] **Step 8: Run the validation test + the filter suite** + +Run: `hatch run test:test-target tests/core/test_compiler.py::TestFilterEngineRejectsInvalidSetRules -v` +Expected: PASS. +Run: `hatch run test:test-target tests/core/test_compiler.py tests/test_engine.py -q` +Expected: PASS — existing filter/set tests unaffected (null-list rules still normalize to wildcard; concrete lists unchanged). If a pre-existing set test used a null rule list and asserted wildcard behaviour, it still passes. + +- [ ] **Step 9: Ruff + commit** + +```bash +hatch run ruff:check src/mountainash_rules/core/compiler.py src/mountainash_rules/engines/filter/engine.py +git add src/mountainash_rules/core/compiler.py src/mountainash_rules/engines/filter/engine.py tests/core/test_compiler.py +git commit -m "feat(filter): set-wildcard ternary short-circuit + reject invalid set rule lists" +``` + +--- + +### Task 4: Accumulator compiler set branches (coalesce / compatible / NA flag) + +**Files:** +- Modify: `src/mountainash_rules/engines/accumulator/compiler.py` +- Test: `tests/accumulator/test_compiler.py` + +**Interfaces:** +- Consumes: `set_wildcard_predicate`, `canonicalize_set_expr`, `sentinel_list_expr` (Task 2). Assumes co_/rhs columns are already normalized (Task 5 guarantees this at ingestion). +- Produces: `SET_MEMBERSHIP`/`SET_EXCLUSION` support in `compile_compatible`, `compile_coalesce`, `compile_coalesce_na_flag`. + +- [ ] **Step 1: Write the failing compiler tests** + +Add to `tests/accumulator/test_compiler.py` (imports `AccumulatorCompiler`, `Dimension`, `MatchStrategy`, `pl` already present; add `from mountainash_rules.core.constants import DataType`): +```python +class TestSetMembershipCompatible: + def _dim(self): + return Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.STR) + + def test_non_empty_intersection_compatible(self, compiler): + expr = compiler.compile_compatible(self._dim()) + df = pl.DataFrame({ + "co_region": pl.Series("co_region", [["AU", "NZ"], ["AU", "NZ"]], dtype=pl.List(pl.Utf8)), + "region_rhs": pl.Series("region_rhs", [["NZ", "UK"], ["US", "CA"]], dtype=pl.List(pl.Utf8)), + }) + out = df.with_columns(expr.alias("c").compile(df, booleanizer=None)) + assert out["c"].to_list() == [True, False] + + def test_wildcard_either_side_compatible(self, compiler): + expr = compiler.compile_compatible(self._dim()) + df = pl.DataFrame({ + "co_region": pl.Series("co_region", [[""], ["AU"]], dtype=pl.List(pl.Utf8)), + "region_rhs": pl.Series("region_rhs", [["US"], [""]], dtype=pl.List(pl.Utf8)), + }) + out = df.with_columns(expr.alias("c").compile(df, booleanizer=None)) + assert out["c"].to_list() == [True, True] + + +class TestSetMembershipCoalesce: + def _dim(self): + return Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.STR) + + def test_intersection_canonicalized(self, compiler): + exprs = compiler.compile_coalesce(self._dim()) + assert len(exprs) == 1 + df = pl.DataFrame({ + "co_region": pl.Series("co_region", [["AU", "NZ", "UK"]], dtype=pl.List(pl.Utf8)), + "region_rhs": pl.Series("region_rhs", [["UK", "NZ", "US"]], dtype=pl.List(pl.Utf8)), + }) + out = df.with_columns(exprs[0].compile(df, booleanizer=None)) + assert out["co_region"].to_list() == [["NZ", "UK"]] # sorted-unique + + def test_wildcard_passthrough(self, compiler): + exprs = compiler.compile_coalesce(self._dim()) + df = pl.DataFrame({ + "co_region": pl.Series("co_region", [[""], ["AU"]], dtype=pl.List(pl.Utf8)), + "region_rhs": pl.Series("region_rhs", [["AU"], [""]], dtype=pl.List(pl.Utf8)), + }) + out = df.with_columns(exprs[0].compile(df, booleanizer=None)) + assert out["co_region"].to_list() == [["AU"], ["AU"]] + + def test_both_wildcard_stays_sentinel(self, compiler): + exprs = compiler.compile_coalesce(self._dim()) + df = pl.DataFrame({ + "co_region": pl.Series("co_region", [[""]], dtype=pl.List(pl.Utf8)), + "region_rhs": pl.Series("region_rhs", [[""]], dtype=pl.List(pl.Utf8)), + }) + out = df.with_columns(exprs[0].compile(df, booleanizer=None)) + assert out["co_region"].to_list() == [[""]] + assert out["co_region"].dtype == pl.List(pl.Utf8) + + +class TestSetExclusionCoalesce: + def _dim(self): + return Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_EXCLUSION, data_type=DataType.STR) + + def test_union_canonicalized(self, compiler): + exprs = compiler.compile_coalesce(self._dim()) + df = pl.DataFrame({ + "co_region": pl.Series("co_region", [["AU", "NZ"]], dtype=pl.List(pl.Utf8)), + "region_rhs": pl.Series("region_rhs", [["NZ", "US"]], dtype=pl.List(pl.Utf8)), + }) + out = df.with_columns(exprs[0].compile(df, booleanizer=None)) + assert out["co_region"].to_list() == [["AU", "NZ", "US"]] + + def test_always_compatible(self, compiler): + expr = compiler.compile_compatible(self._dim()) + df = pl.DataFrame({ + "co_region": pl.Series("co_region", [["AU"], [""]], dtype=pl.List(pl.Utf8)), + "region_rhs": pl.Series("region_rhs", [["NZ"], ["US"]], dtype=pl.List(pl.Utf8)), + }) + out = df.with_columns(expr.alias("c").compile(df, booleanizer=None)) + assert out["c"].to_list() == [True, True] + + +class TestSetNaFlag: + def _dim(self, strategy): + return Dimension(dimension_name="region", match_strategy=strategy, data_type=DataType.STR) + + def test_na_flag_from_final_coalesced_value(self, compiler): + # wildcard+wildcard -> 1 ; wildcard+concrete -> 0 ; concrete+concrete -> 0 + expr = compiler.compile_coalesce_na_flag(self._dim(MatchStrategy.SET_MEMBERSHIP)) + df = pl.DataFrame({ + "co_region": pl.Series("co_region", [[""], [""], ["AU"]], dtype=pl.List(pl.Utf8)), + "region_rhs": pl.Series("region_rhs", [[""], ["AU"], ["NZ"]], dtype=pl.List(pl.Utf8)), + }) + out = df.with_columns(expr.compile(df, booleanizer=None)) + assert out["co_region_na"].to_list() == [1, 0, 0] +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `hatch run test:test-target tests/accumulator/test_compiler.py::TestSetMembershipCompatible tests/accumulator/test_compiler.py::TestSetMembershipCoalesce tests/accumulator/test_compiler.py::TestSetExclusionCoalesce tests/accumulator/test_compiler.py::TestSetNaFlag -v` +Expected: FAIL — `ValueError: Strategy SET_MEMBERSHIP not supported by accumulator` (compatible/coalesce), and the NA-flag `else` branch mis-computes for lists. + +- [ ] **Step 3: Add the imports and the set branches** + +In `src/mountainash_rules/engines/accumulator/compiler.py`, add to the imports: +```python +from mountainash_rules.core.set_wildcard import ( + set_wildcard_predicate, + canonicalize_set_expr, + sentinel_list_expr, +) +``` +Add a `case` to `compile_compatible`'s `match` (before `case _:`): +```python + case MatchStrategy.SET_MEMBERSHIP: + return self._compatible_set_membership(dim) + case MatchStrategy.SET_EXCLUSION: + return ma.lit(True) +``` +Add a `case` to `compile_coalesce`'s `match` (before `case _:`): +```python + case MatchStrategy.SET_MEMBERSHIP: + return self._coalesce_set(dim, "intersection") + case MatchStrategy.SET_EXCLUSION: + return self._coalesce_set(dim, "union") +``` +Change `compile_coalesce_na_flag` to insert a set branch. Replace its body with: +```python + def compile_coalesce_na_flag(self, dim: Dimension) -> BaseExpressionAPI: + """Expression for the coalesced NA flag (1 = combination leaves dim unconstrained).""" + if dim.match_strategy == MatchStrategy.RANGE: + co_min_s, co_max_s, rhs_min_s, rhs_max_s = self._range_sentinel_checks(dim) + all_sentinel = ( + co_min_s.__and__(rhs_min_s) + .__and__(co_max_s) + .__and__(rhs_max_s) + ) + return all_sentinel.cast(int).alias(f"co_{dim.dimension_name}_na") + if dim.match_strategy in (MatchStrategy.SET_MEMBERSHIP, MatchStrategy.SET_EXCLUSION): + co_w, rhs_w = self._set_wild_checks(dim) + field = dim.resolved_rule_field + return co_w.__and__(rhs_w).cast(int).alias(f"co_{field}_na") + co_sentinel, rhs_sentinel = self._sentinel_checks(dim) + field = dim.resolved_rule_field + return co_sentinel.__and__(rhs_sentinel).cast(int).alias(f"co_{field}_na") +``` +> The set NA flag is `co_wild AND rhs_wild`, which equals the wildcard status of the FINAL coalesced value: coalesce yields `[sentinel]` iff both inputs are wildcard (wildcard+concrete → the concrete side; concrete+concrete → intersection/union). This satisfies spec F8 while remaining computable from the LHS/RHS inputs in the same `with_columns` pass. + +Add the two helper methods at the end of the class: +```python + def _set_wild_checks(self, dim: Dimension) -> tuple[BaseExpressionAPI, BaseExpressionAPI]: + field = dim.resolved_rule_field + co_w = set_wildcard_predicate(dim, ma.col(f"co_{field}")) + rhs_w = set_wildcard_predicate(dim, ma.col(f"{field}_rhs")) + return co_w, rhs_w + + def _compatible_set_membership(self, dim: Dimension) -> BaseExpressionAPI: + co_w, rhs_w = self._set_wild_checks(dim) + field = dim.resolved_rule_field + intersection_nonempty = ( + ma.col(f"co_{field}") + .list.set_intersection(ma.col(f"{field}_rhs")) + .list.len() + .gt(ma.lit(0)) + ) + return co_w.__or__(rhs_w).__or__(intersection_nonempty) + + def _coalesce_set(self, dim: Dimension, op: str) -> list[BaseExpressionAPI]: + co_w, rhs_w = self._set_wild_checks(dim) + field = dim.resolved_rule_field + co = ma.col(f"co_{field}") + rhs = ma.col(f"{field}_rhs") + combined = ( + co.list.set_intersection(rhs) if op == "intersection" else co.list.set_union(rhs) + ) + new_val = ( + ma.when(co_w.__and__(rhs_w)).then(sentinel_list_expr(dim)) + .when(co_w).then(rhs) + .when(rhs_w).then(co) + .otherwise(canonicalize_set_expr(combined)) + .alias(f"co_{field}") + ) + return [new_val] +``` + +- [ ] **Step 4: Run the compiler tests to verify they pass** + +Run: `hatch run test:test-target tests/accumulator/test_compiler.py::TestSetMembershipCompatible tests/accumulator/test_compiler.py::TestSetMembershipCoalesce tests/accumulator/test_compiler.py::TestSetExclusionCoalesce tests/accumulator/test_compiler.py::TestSetNaFlag -v` +Expected: PASS. + +- [ ] **Step 5: Run accumulator compiler suite + purity + ruff** + +Run: `hatch run test:test-target tests/accumulator/test_compiler.py tests/test_backend_purity.py -q` +Expected: PASS. +Run: `hatch run ruff:check src/mountainash_rules/engines/accumulator/compiler.py` +Expected: All checks passed. + +- [ ] **Step 6: Commit** + +```bash +git add src/mountainash_rules/engines/accumulator/compiler.py tests/accumulator/test_compiler.py +git commit -m "feat(accumulator): set coalescing/compatible/NA-flag on in-band sentinel" +``` + +--- + +### Task 5: Accumulator engine — normalization stage + seed NA + pre-frontier assertion (F5) + +**Files:** +- Modify: `src/mountainash_rules/engines/accumulator/engine.py` (`build`, new `_normalize_set_columns`, `_create_anchor` NA loop, pre-frontier assertion) +- Test: `tests/accumulator/test_engine.py` + +**Interfaces:** +- Consumes: `validate_set_columns`, `normalize_set_expr`, `set_wildcard_predicate` (Task 2); the accumulator compiler set branches (Task 4). +- Produces: `build(rules)` with any set dimension produces a lattice whose set `co_` columns are non-null and canonical; the frontier dedupes wildcard combinations. + +- [ ] **Step 1: Write the failing build regression tests** + +Add to `tests/accumulator/test_engine.py` (helpers `_rows`, `relation` already imported; add `from mountainash_rules.core.constants import DataType` if absent): +```python +class TestSetMembershipBuildFrontier: + def _metadata(self): + return DimensionsMetadata(dimensions=[ + Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.STR), + ]) + + def test_three_wildcard_rules_collapse_to_single_maximal(self): + # THE anchor regression: 3 wildcard-set rules must dedupe to pp=30, NOT 7 combos. + rules = pl.DataFrame({ + "rule_name": ["R1", "R2", "R3"], + "region": pl.Series("region", [None, None, None], dtype=pl.List(pl.Utf8)), + }) + engine = AccumulatorEngine(dimension_metadata=self._metadata()) + rows = _rows(engine.build(rules).combinations) + assert set(rows["__prime_product"]) == {30} + + def test_two_membership_rules_coalesce_to_intersection(self): + rules = pl.DataFrame({ + "rule_name": ["R1", "R2"], + "region": pl.Series("region", [["AU", "NZ", "UK"], ["NZ", "UK", "US"]], dtype=pl.List(pl.Utf8)), + }) + engine = AccumulatorEngine(dimension_metadata=self._metadata()) + rows = _rows(engine.build(rules).combinations) + by_pp = dict(zip(rows["__prime_product"], rows["co_region"])) + assert sorted(by_pp[6]) == ["NZ", "UK"] + + def test_same_set_different_order_dedupes(self): + # Ordering: two rules whose sets are equal up to order must produce one combo. + rules = pl.DataFrame({ + "rule_name": ["R1", "R2"], + "region": pl.Series("region", [["UK", "NZ"], ["NZ", "UK"]], dtype=pl.List(pl.Utf8)), + }) + engine = AccumulatorEngine(dimension_metadata=self._metadata()) + rows = _rows(engine.build(rules).combinations) + # {R1,R2} intersection == {NZ,UK}; both singletons canonicalize equal, so + # the maximal pp=6 combination is present and carries the canonical set. + by_pp = dict(zip(rows["__prime_product"], rows["co_region"])) + assert by_pp[6] == ["NZ", "UK"] + + +class TestSetExclusionBuild: + def _metadata(self): + return DimensionsMetadata(dimensions=[ + Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_EXCLUSION, data_type=DataType.STR), + ]) + + def test_two_exclusion_rules_coalesce_to_union(self): + rules = pl.DataFrame({ + "rule_name": ["R1", "R2"], + "region": pl.Series("region", [["AU", "NZ"], ["NZ", "US"]], dtype=pl.List(pl.Utf8)), + }) + engine = AccumulatorEngine(dimension_metadata=self._metadata()) + rows = _rows(engine.build(rules).combinations) + by_pp = dict(zip(rows["__prime_product"], rows["co_region"])) + assert by_pp[6] == ["AU", "NZ", "US"] + + +class TestSetBuildValidation: + def _metadata(self): + return DimensionsMetadata(dimensions=[ + Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.STR), + ]) + + def test_embedded_sentinel_rejected(self): + import pytest + rules = pl.DataFrame({ + "rule_name": ["R1"], + "region": pl.Series("region", [["AU", ""]], dtype=pl.List(pl.Utf8)), + }) + engine = AccumulatorEngine(dimension_metadata=self._metadata()) + with pytest.raises(ValueError, match="sentinel"): + engine.build(rules) +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `hatch run test:test-target tests/accumulator/test_engine.py::TestSetMembershipBuildFrontier tests/accumulator/test_engine.py::TestSetExclusionBuild tests/accumulator/test_engine.py::TestSetBuildValidation -v` +Expected: FAIL — the accumulator does not yet normalize set columns; the compiler set branches exist (Task 4) but `co_region` starts null for wildcards, so `test_three_wildcard_rules_collapse_to_single_maximal` returns `{2,3,5,6,10,15,30}`, and `TestSetBuildValidation` does not raise. + +- [ ] **Step 3: Add the normalization stage import** + +In `src/mountainash_rules/engines/accumulator/engine.py`, add near the other `mountainash_rules.core` imports: +```python +from mountainash_rules.core.set_wildcard import ( + validate_set_columns, + normalize_set_expr, + set_wildcard_predicate, +) +``` + +- [ ] **Step 4: Add the `_normalize_set_columns` method and call it in `build`** + +Add this method to `AccumulatorEngine` (near `_create_anchor`): +```python + def _set_dims(self) -> list[Dimension]: + return [ + d for d in self._constraint_dims + if d.match_strategy in (MatchStrategy.SET_MEMBERSHIP, MatchStrategy.SET_EXCLUSION) + ] + + def _normalize_set_columns(self, rules_pl: t.Any) -> t.Any: + """Validate + normalize every set-dimension rule column to the non-null, + canonical in-band-sentinel form. Runs for EVERY build path before the + empty-frame branch and the anchor, so set co_ columns are never null.""" + set_dims = self._set_dims() + if not set_dims: + return rules_pl + rel = relation(rules_pl) + validate_set_columns(rel, set_dims) + rel = rel.with_columns(*[ + normalize_set_expr(dim, ma.col(dim.resolved_rule_field)).alias(dim.resolved_rule_field) + for dim in set_dims + ]) + return rel.to_polars() +``` +In `build`, right after `rules_pl = rel.to_polars()` (currently `engine.py:122`) and BEFORE `n_rules = len(rules_pl)`: +```python + rules_pl = self._normalize_set_columns(rules_pl) +``` + +- [ ] **Step 5: Fix the seed NA loop for set dimensions** + +In `_create_anchor`, the `# Add NA flag columns` loop currently does `ma.col(field).eq(ma.lit(sentinel))` for the non-range branch — wrong for list columns. Replace the non-range `else` branch of that loop so set dims use `set_wildcard_predicate`: +```python + else: + field = dim.resolved_rule_field + if dim.match_strategy in (MatchStrategy.SET_MEMBERSHIP, MatchStrategy.SET_EXCLUSION): + na_exprs.append( + set_wildcard_predicate(dim, ma.col(field)) + .cast(int) + .alias(f"co_{field}_na") + ) + else: + sentinel = unknown_sentinel_for(dim.data_type) + na_exprs.append( + ma.col(field).eq(ma.lit(sentinel)) + .cast(int) + .alias(f"co_{field}_na") + ) +``` +(The `co_` value loop above it needs no change — for a set dim it copies the already-normalized `field` into `co_{field}`.) + +- [ ] **Step 6: Add the pre-frontier non-null assertion** + +In `build`, replace the frontier line `result = self._frontier_filter(all_combos)` with a guarded version: +```python + self._assert_set_columns_non_null(all_combos) + result = self._frontier_filter(all_combos) +``` +Add the assertion method: +```python + def _assert_set_columns_non_null(self, all_combos: t.Any) -> None: + """Safety net for the frontier 'no change' invariant: every set co_ column + must be non-null before the dominance self-join (null keys silently defeat + pruning). Runs only when set dims are present.""" + set_dims = self._set_dims() + if not set_dims: + return + cols = [f"co_{d.resolved_rule_field}" for d in set_dims] + checked = relation(all_combos).with_columns(*[ + ma.col(c).is_null().cast(int).alias(f"__null_{c}") for c in cols + ]).collect() + for c in cols: + # collected native frame; builtin sum over the flag column + if sum(checked[f"__null_{c}"]) > 0: + raise AssertionError( + f"set co_ column {c!r} contains null before frontier filter — " + f"normalization did not reach every build path" + ) +``` +> `checked[f"__null_{c}"]` indexes a collected native frame (polars/pandas); `sum(...)` over its values is backend-pure (builtin). This is a cheap invariant guard, not hot-path code. + +- [ ] **Step 7: Run the build tests to verify they pass** + +Run: `hatch run test:test-target tests/accumulator/test_engine.py::TestSetMembershipBuildFrontier tests/accumulator/test_engine.py::TestSetExclusionBuild tests/accumulator/test_engine.py::TestSetBuildValidation -v` +Expected: PASS — `test_three_wildcard_rules_collapse_to_single_maximal` now returns `{30}`. + +- [ ] **Step 8: Full accumulator suite + purity + ruff** + +Run: `hatch run test:test-target tests/accumulator/ tests/test_backend_purity.py -q` +Expected: PASS. +Run: `hatch run ruff:check src/mountainash_rules/engines/accumulator/engine.py` +Expected: All checks passed. + +- [ ] **Step 9: Commit** + +```bash +git add src/mountainash_rules/engines/accumulator/engine.py tests/accumulator/test_engine.py +git commit -m "feat(accumulator): set-column normalization stage; frontier untouched, dedupes correctly" +``` + +--- + +### Task 6: Apply round-trip, float typing, idempotence, docs + +**Files:** +- Test: `tests/accumulator/test_apply.py` (apply round-trip) +- Test: `tests/accumulator/test_engine.py` (float-dim build) +- Modify: `CLAUDE.md` (match-strategy note) + +**Interfaces:** +- Consumes: everything from Tasks 1–5. +- Produces: end-to-end verification + docs. Nothing downstream. + +- [ ] **Step 1: Write the failing apply round-trip test** + +Add to `tests/accumulator/test_apply.py` (uses `AccumulatorEngine`, `Dimension`, `DimensionsMetadata`, `_rows`, `pl` already present; add `from mountainash_rules.core.constants import DataType` if absent): +```python +class TestSetMembershipApply: + def _metadata(self): + return DimensionsMetadata(dimensions=[ + Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.STR), + ]) + + def test_context_in_intersection_matches_combination(self): + rules = pl.DataFrame({ + "rule_name": ["R1", "R2"], + "region": pl.Series("region", [["AU", "NZ", "UK"], ["NZ", "UK", "US"]], dtype=pl.List(pl.Utf8)), + }) + engine = AccumulatorEngine(dimension_metadata=self._metadata()) + lattice = engine.build(rules) + result = engine.apply(lattice, {"region": "NZ"}) + assert 6 in set(_rows(result.provenance)["__prime_product"]) + + def test_wildcard_combination_matches_any_context_exact_count(self): + rules = pl.DataFrame({ + "rule_name": ["R1"], + "region": pl.Series("region", [None], dtype=pl.List(pl.Utf8)), + }) + engine = AccumulatorEngine(dimension_metadata=self._metadata()) + lattice = engine.build(rules) + result = engine.apply(lattice, {"region": "ANYTHING"}) + assert result.count == 1 # exactly the single wildcard combination +``` +> Accessors verified against `result.py`/existing `test_apply.py`: `result.provenance` (relation with `__prime_product`, read via `_rows`), `result.count`. No `.combinations` on the apply result. `apply` accepts a dict. This runs on polars (build polars-internal; apply on polars). If a future cross-backend apply sweep trips narwhals list ops, extend the existing `mountainash#89` entry in `tests/conftest.py` `_UPSTREAM_XFAILS` — never a new xfail group. + +- [ ] **Step 2: Run to verify it passes** + +Run: `hatch run test:test-target tests/accumulator/test_apply.py::TestSetMembershipApply -v` +Expected: PASS (build + apply already implemented by Tasks 2–5; this is end-to-end verification). + +- [ ] **Step 3: Write the float-dimension build test** + +Add to `tests/accumulator/test_engine.py`: +```python +class TestFloatSetDimensionBuild: + def _metadata(self): + return DimensionsMetadata(dimensions=[ + Dimension(dimension_name="scores", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.FLOAT), + ]) + + def test_float_set_wildcard_and_coalesce(self): + rules = pl.DataFrame({ + "rule_name": ["R1", "R2"], + "scores": pl.Series("scores", [[1.5, 2.5, 3.5], None], dtype=pl.List(pl.Float64)), + }) + engine = AccumulatorEngine(dimension_metadata=self._metadata()) + lattice = engine.build(rules) + rows = _rows(lattice.combinations) + by_pp = dict(zip(rows["__prime_product"], rows["co_scores"])) + # R2 is a wildcard; {R1,R2} coalesces to R1's concrete set (wildcard passthrough). + assert by_pp[6] == [1.5, 2.5, 3.5] + # Column stays a Float list — verify no dtype collapse. + import polars as _pl + mat = relation(lattice.combinations).to_polars() + assert mat.schema["co_scores"] == _pl.List(_pl.Float64) +``` + +- [ ] **Step 4: Run the float test** + +Run: `hatch run test:test-target tests/accumulator/test_engine.py::TestFloatSetDimensionBuild -v` +Expected: PASS. + +- [ ] **Step 5: Update `CLAUDE.md`** + +In `CLAUDE.md`, the Match Strategies table row for `set_membership` / `set_exclusion` currently reads (from the A2 merge): +``` +| `set_membership` / `set_exclusion` | list column | Polars-native fallback | +``` +Replace with: +``` +| `set_membership` / `set_exclusion` | list column | Polars-native fallback; accumulator-coalesceable (membership → list intersection, exclusion → list union). Wildcard = in-band `[unknown_sentinel_for(dtype)]` (never null; bool unsupported); see `null-is-not-a-portable-sentinel` principle. | +``` +And under the "Ternary match logic" / sentinels area, add one line noting that set-dimension wildcards use the in-band `[sentinel]` list (not null), normalized at ingestion in both engines. + +- [ ] **Step 6: Full quick suite + purity** + +Run: `hatch run test:test-quick` +Expected: PASS (no new failures; existing xfails unchanged). +Run: `hatch run test:test-target tests/test_backend_purity.py -q` +Expected: PASS (three `# allow:` tags unchanged; no new native imports). + +- [ ] **Step 7: Commit** + +```bash +git add tests/accumulator/test_apply.py tests/accumulator/test_engine.py CLAUDE.md +git commit -m "test+docs(accumulator): set-wildcard apply round-trip, float typing; document in-band sentinel" +``` + +--- + +## Notes for the executor + +- **The frontier bug fix is Task 5's normalization stage, not any change to `_frontier_filter`.** If you find yourself editing `_frontier_filter`, `_check_overflow`, or partition routing, stop — the design is explicit that these stay untouched. The fix is that their inputs are now null-free and canonical. +- **Never construct a set value with a bare `ma.lit([sent])` or a native list-dtype cast.** Always `sentinel_list_expr(dim)` (element-typed literal). A `.cast(pl.List(...))` needs `import polars` (purity failure) and is refused by the backend anyway. +- **Set `co_` columns must never be null.** Normalization at ingestion (Task 5) guarantees it; the pre-frontier assertion (Task 5 Step 6) is the safety net. `set_wildcard_predicate` and the coalesce branches assume non-null input. +- **Backend purity is the sharpest tripwire.** `validate_set_columns` and `_assert_set_columns_non_null` use `relation(...).collect()` + builtin `len`/`sum` — never `import polars`. +- **`co__na` = `co_wild AND rhs_wild`** (Task 4) — equals the wildcard status of the final coalesced value, computed from inputs in the same pass. From 9ef54a77911c913cd77f3cf7871cb27f1d76965f Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 20 Jul 2026 12:43:00 +1000 Subject: [PATCH 04/10] docs: revise A1 plan+spec after Codex plan review (gpt-5.6-sol) Two confirmed bugs fixed + minors: - split validation: portable reservation check (contains+len, both engines) vs null-element check (drop_nulls, Ibis-unsupported -> accumulator build only) - filter validation via shared _validate_set_rules_once() called in BOTH _scored_relation AND _evaluate_batch_frame (batch path was bypassed) - F8 spec permits proven-equivalent co_wild AND rhs_wild input formula - fix fail-first expected output; dedup test asserts exact {6}; empty-path assertion vacuous note; docs backend scope (not 'polars-native fallback') - cross-dtype (float/date) idempotence test; pin tests/filter/test_engine.py Co-Authored-By: Claude Opus 4.8 (1M context) --- ...07-20-accumulator-set-sentinel-wildcard.md | 190 ++++++++++++------ ...ccumulator-set-sentinel-wildcard-design.md | 9 +- 2 files changed, 141 insertions(+), 58 deletions(-) diff --git a/docs/superpowers/plans/2026-07-20-accumulator-set-sentinel-wildcard.md b/docs/superpowers/plans/2026-07-20-accumulator-set-sentinel-wildcard.md index a9cd05f..4808781 100644 --- a/docs/superpowers/plans/2026-07-20-accumulator-set-sentinel-wildcard.md +++ b/docs/superpowers/plans/2026-07-20-accumulator-set-sentinel-wildcard.md @@ -119,7 +119,8 @@ git commit -m "feat(dimension): reject bool set dimensions (no typed wildcard se - `canonicalize_set_expr(col: BaseExpressionAPI) -> BaseExpressionAPI` — sort + unique. - `normalize_set_expr(dim, col) -> BaseExpressionAPI` — null→`[sentinel]`, concrete→sorted-unique. - `set_wildcard_predicate(dim, col) -> BaseExpressionAPI` — `col.list.contains(sentinel)` (post-normalization). - - `validate_set_columns(rules_rel, set_dims: list[Dimension]) -> None` — raises `ValueError` on embedded-sentinel or element-null rule lists. + - `validate_set_columns(rules_rel, set_dims: list[Dimension]) -> None` — **portable** (both engines); raises `ValueError` on embedded-sentinel lists. Uses only `list.contains`/`list.len`. + - `validate_set_no_null_elements(rules_rel, set_dims: list[Dimension]) -> None` — **accumulator build only** (uses `list.drop_nulls`, Ibis-unsupported); raises `ValueError` on element-null lists. - Consumes: `unknown_sentinel_for`, `DataType` from `core/constants.py`; `Dimension` from `core/dimension.py`. - [ ] **Step 1: Write the failing helper tests** @@ -139,6 +140,7 @@ from mountainash_rules.core.set_wildcard import ( normalize_set_expr, set_wildcard_predicate, validate_set_columns, + validate_set_no_null_elements, ) from mountainash.relations import relation @@ -189,6 +191,26 @@ class TestNormalizeAndDetect: twice = once.with_columns(normalize_set_expr(dim, ma.col("region")).alias("region").compile(once, booleanizer=None)) assert once["region"].to_list() == twice["region"].to_list() + def test_normalize_idempotent_float_and_date(self): + # F9: idempotence must hold across dtypes, not just str. + import datetime as _dt + import mountainash as ma + from mountainash_rules.core.constants import DataType + float_dim = _float_dim() + fdf = pl.DataFrame({"scores": pl.Series("scores", [None, [2.5, 1.5]], dtype=pl.List(pl.Float64))}) + f1 = fdf.with_columns(normalize_set_expr(float_dim, ma.col("scores")).alias("scores").compile(fdf, booleanizer=None)) + f2 = f1.with_columns(normalize_set_expr(float_dim, ma.col("scores")).alias("scores").compile(f1, booleanizer=None)) + assert f1["scores"].to_list() == f2["scores"].to_list() + assert f1["scores"].to_list() == [[-999999999.0], [1.5, 2.5]] + + date_dim = Dimension(dimension_name="days", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.DATE) + d = [_dt.date(2020, 1, 2), _dt.date(2020, 1, 1)] + ddf = pl.DataFrame({"days": pl.Series("days", [None, d], dtype=pl.List(pl.Date))}) + d1 = ddf.with_columns(normalize_set_expr(date_dim, ma.col("days")).alias("days").compile(ddf, booleanizer=None)) + d2 = d1.with_columns(normalize_set_expr(date_dim, ma.col("days")).alias("days").compile(d1, booleanizer=None)) + assert d1["days"].to_list() == d2["days"].to_list() + assert d1["days"].dtype == pl.List(pl.Date) + class TestCanonicalize: def test_sort_and_dedupe(self): @@ -205,19 +227,26 @@ class TestValidateSetColumns: with pytest.raises(ValueError, match="sentinel"): validate_set_columns(relation(rules), [dim]) - def test_null_element_rejected(self): + def test_embedded_sentinel_passes_null_element_check(self): + # validate_set_columns is reservation-only; whole-list null + concrete OK. + dim = _str_dim() + rules = pl.DataFrame({"region": pl.Series("region", [None, ["AU"], [""]], dtype=pl.List(pl.Utf8))}) + validate_set_columns(relation(rules), [dim]) # no raise + + def test_null_element_rejected_by_dedicated_check(self): dim = _str_dim() rules = pl.DataFrame({"region": pl.Series("region", [["AU", None]], dtype=pl.List(pl.Utf8))}) with pytest.raises(ValueError, match="null element"): - validate_set_columns(relation(rules), [dim]) + validate_set_no_null_elements(relation(rules), [dim]) - def test_whole_list_null_is_allowed(self): + def test_null_element_check_allows_whole_list_null(self): dim = _str_dim() - rules = pl.DataFrame({"region": pl.Series("region", [None, ["AU"], [""]], dtype=pl.List(pl.Utf8))}) - validate_set_columns(relation(rules), [dim]) # no raise + rules = pl.DataFrame({"region": pl.Series("region", [None, ["AU"]], dtype=pl.List(pl.Utf8))}) + validate_set_no_null_elements(relation(rules), [dim]) # no raise def test_no_set_dims_is_noop(self): validate_set_columns(relation(pl.DataFrame({"x": [1]})), []) + validate_set_no_null_elements(relation(pl.DataFrame({"x": [1]})), []) ``` - [ ] **Step 2: Run to verify it fails** @@ -302,7 +331,12 @@ def _embedded_sentinel_predicate(dim: Dimension, field: str) -> BaseExpressionAP def _null_element_predicate(field: str) -> BaseExpressionAPI: - """True for a non-null list that contains a null element.""" + """True for a non-null list that contains a null element. + + Uses ``list.drop_nulls`` — mountainash's IBIS backend raises + ``BackendCapabilityError`` for this op, so callers must only run this on the + polars-internal accumulator build path (see ``validate_set_no_null_elements``). + """ col = ma.col(field) length = col.list.len() non_null_length = col.list.drop_nulls().list.len() @@ -310,13 +344,12 @@ def _null_element_predicate(field: str) -> BaseExpressionAPI: def validate_set_columns(rules_rel: t.Any, set_dims: list[Dimension]) -> None: - """Raise ``ValueError`` if any set-dimension rule list is a data error. + """Raise ``ValueError`` if a concrete set-rule list embeds the reserved sentinel. - A data error is (a) a concrete list embedding the sentinel among other - values, or (b) a list with a null element. A whole-list null is valid — it is - the wildcard. ``rules_rel`` is a ``mountainash`` relation; the check collects - the offending rows and uses builtin ``len`` (backend-pure — ``collect`` - returns a native frame that supports ``len``). + Portable — uses only ``list.contains`` + ``list.len`` (Ibis/Narwhals-safe), so + it runs in BOTH engines on any backend. A whole-list null is valid (the + wildcard). ``rules_rel`` is a ``mountainash`` relation; ``collect`` returns a + native frame that supports builtin ``len`` (backend-pure — no native import). """ for dim in set_dims: field = dim.resolved_rule_field @@ -327,6 +360,18 @@ def validate_set_columns(rules_rel: t.Any, set_dims: list[Dimension]) -> None: f"reserved wildcard sentinel {unknown_sentinel_for(dim.data_type)!r}. " f"The sentinel is only valid as the sole element (the wildcard)." ) + + +def validate_set_no_null_elements(rules_rel: t.Any, set_dims: list[Dimension]) -> None: + """Raise ``ValueError`` if a set-rule list contains a null element. + + Uses ``list.drop_nulls`` (Ibis-unsupported), so this is called ONLY on the + polars-internal accumulator build path. The filter engine does not call it: + ``t_is_in`` tolerates a null element (it matches nothing), so a standalone + filter engine over Ibis set rules is unaffected. + """ + for dim in set_dims: + field = dim.resolved_rule_field null_elem = rules_rel.filter(_null_element_predicate(field)).collect() if len(null_elem) > 0: raise ValueError( @@ -359,12 +404,12 @@ git commit -m "feat(core): shared in-band sentinel set-wildcard helpers + valida **Files:** - Modify: `src/mountainash_rules/core/compiler.py` (`_compile_set_membership`, `_compile_set_exclusion`) -- Modify: `src/mountainash_rules/engines/filter/engine.py` (validate set columns once) -- Test: `tests/core/test_compiler.py` (filter compiler ternary tests) +- Modify: `src/mountainash_rules/engines/filter/engine.py` (validate set columns once — both single and batch paths) +- Test: `tests/core/test_compiler.py` (filter compiler ternary tests); `tests/filter/test_engine.py` (engine validation, single + batch) **Interfaces:** - Consumes: `normalize_set_expr`, `set_wildcard_predicate`, `validate_set_columns` (Task 2). -- Produces: filter set ternaries that short-circuit the wildcard to 0 and normalize null input; the filter engine rejects invalid set rule lists. +- Produces: filter set ternaries that short-circuit the wildcard to 0 and normalize null input; the filter engine rejects embedded-sentinel set rule lists on **both** `evaluate`/`explain` (via `_scored_relation`) and `evaluate_batch` (via `_evaluate_batch_frame`). - [ ] **Step 1: Write the failing filter compiler tests** @@ -493,33 +538,49 @@ Replace `_compile_set_membership` and `_compile_set_exclusion` with: Run: `hatch run test:test-target tests/core/test_compiler.py::TestSetMembershipTernary tests/core/test_compiler.py::TestSetExclusionTernary -v` Expected: PASS. -- [ ] **Step 5: Write the failing filter-engine validation test** +- [ ] **Step 5: Write the failing filter-engine validation tests (single AND batch)** -Add to `tests/core/test_compiler.py` (or wherever the filter engine is tested — use `tests/test_engine.py` if that is the filter-engine suite; this test uses the public `ExpressionRulesEngine`): +Add to `tests/filter/test_engine.py` (the filter-engine suite; uses the public `ExpressionRulesEngine`): ```python class TestFilterEngineRejectsInvalidSetRules: - def test_embedded_sentinel_rule_rejected_on_evaluate(self): - import polars as pl - import pytest - from mountainash_rules import ExpressionRulesEngine, Dimension, DimensionsMetadata + def _meta(self): + from mountainash_rules import Dimension, DimensionsMetadata from mountainash_rules.core.constants import MatchStrategy, DataType - meta = DimensionsMetadata(dimensions=[ + return DimensionsMetadata(dimensions=[ Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.STR), ]) - rules = pl.DataFrame({ + + def _rules(self): + import polars as pl + return pl.DataFrame({ "rule_name": ["R1"], "region": pl.Series("region", [["AU", ""]], dtype=pl.List(pl.Utf8)), }) - engine = ExpressionRulesEngine(rules=rules, dimension_metadata=meta) + + def test_embedded_sentinel_rejected_on_evaluate(self): + import pytest + from mountainash_rules import ExpressionRulesEngine + engine = ExpressionRulesEngine(rules=self._rules(), dimension_metadata=self._meta()) with pytest.raises(ValueError, match="sentinel"): engine.evaluate({"region": "AU"}) + + def test_embedded_sentinel_rejected_on_evaluate_batch(self): + # evaluate_batch does NOT route through _scored_relation — this covers the + # batch path explicitly (a batch-first call must still validate). + import polars as pl + import pytest + from mountainash_rules import ExpressionRulesEngine + engine = ExpressionRulesEngine(rules=self._rules(), dimension_metadata=self._meta()) + contexts = pl.DataFrame({"region": ["AU"]}) + with pytest.raises(ValueError, match="sentinel"): + engine.evaluate_batch(contexts) ``` -> Verified: `ExpressionRulesEngine(rules=..., dimension_metadata=...)` (`engine.py:55`); `evaluate` accepts a dict. Both `ExpressionRulesEngine` and `Dimension`/`DimensionsMetadata` are public root imports. +> Verified: `ExpressionRulesEngine(rules=..., dimension_metadata=...)` (`engine.py:55`); `evaluate` accepts a dict; `evaluate_batch(contexts)` accepts a frame. All three names are public root imports. Check `tests/filter/test_engine.py`'s existing top-of-file imports and reuse them where present. - [ ] **Step 6: Run to verify it fails** -Run: `hatch run test:test-target tests/core/test_compiler.py::TestFilterEngineRejectsInvalidSetRules -v` -Expected: FAIL — no `ValueError` raised (invalid list silently treated as wildcard). +Run: `hatch run test:test-target tests/filter/test_engine.py::TestFilterEngineRejectsInvalidSetRules -v` +Expected: BOTH FAIL — no `ValueError` raised (invalid list silently treated as wildcard) on either the single or batch path. - [ ] **Step 7: Add memoized validation to the filter engine** @@ -528,27 +589,40 @@ In `src/mountainash_rules/engines/filter/engine.py`, add the import near the oth from mountainash_rules.core.set_wildcard import validate_set_columns from mountainash_rules.core.constants import MatchStrategy ``` -In `_scored_relation` (around line 407–412), immediately after it builds the rules relation (`rel = relation(self._rules)`), add a one-time validation guarded by a flag. First, in `__init__` (near `self._rules = rules`, line 74), add: +In `__init__` (near `self._rules = rules`, line 74), add the memo flag: ```python self._set_dims_validated = False ``` -Then in `_scored_relation`, right after `rel = relation(self._rules)`: +Add a shared one-time validation method: ```python - if self._metadata is not None and not self._set_dims_validated: - set_dims = [ - d for d in self._metadata.dimensions - if d.match_strategy in (MatchStrategy.SET_MEMBERSHIP, MatchStrategy.SET_EXCLUSION) - ] - validate_set_columns(rel, set_dims) - self._set_dims_validated = True + def _validate_set_rules_once(self) -> None: + """Reject set rule lists that embed the reserved sentinel — once, portably. + + Called from BOTH scoring entry points (single and batch) because + evaluate_batch does not route through _scored_relation. metadata is None + when the engine was built from dimension_expressions (no Dimension objects + to inspect), so skip that case. + """ + if self._metadata is None or self._set_dims_validated: + return + set_dims = [ + d for d in self._metadata.dimensions + if d.match_strategy in (MatchStrategy.SET_MEMBERSHIP, MatchStrategy.SET_EXCLUSION) + ] + validate_set_columns(relation(self._rules), set_dims) + self._set_dims_validated = True +``` +Call it at the top of **`_scored_relation`** (line 407) and at the top of **`_evaluate_batch_frame`** (line 303) — both, because `evaluate_batch` → `_evaluate_batch_frame` (line 248/259) builds its own `rules_rel = relation(self._rules)` at line 313 and never calls `_scored_relation`. In each, add as the first statement: +```python + self._validate_set_rules_once() ``` -> Verified: metadata attribute is `self._metadata` (`engine.py:69`), and it is **`None`** when the engine was constructed from `dimension_expressions` instead of metadata — hence the `self._metadata is not None` guard (the expressions path has no `Dimension` objects to inspect). `_scored_relation` is shared by `evaluate`, `explain`, and `evaluate_batch`, so validating here covers all entry points. +> Verified: metadata attribute is `self._metadata` (`engine.py:69`), `None` on the `dimension_expressions` path. `evaluate`/`explain` go through `_scored_relation`; `evaluate_batch` goes through `_evaluate_batch_frame`. Calling `_validate_set_rules_once` in both covers every entry point; the memo flag makes repeat calls cheap. `relation` is already imported in this module. -- [ ] **Step 8: Run the validation test + the filter suite** +- [ ] **Step 8: Run the validation tests + the filter suite** -Run: `hatch run test:test-target tests/core/test_compiler.py::TestFilterEngineRejectsInvalidSetRules -v` -Expected: PASS. -Run: `hatch run test:test-target tests/core/test_compiler.py tests/test_engine.py -q` +Run: `hatch run test:test-target tests/filter/test_engine.py::TestFilterEngineRejectsInvalidSetRules -v` +Expected: PASS (both single and batch). +Run: `hatch run test:test-target tests/core/test_compiler.py tests/filter/ -q` Expected: PASS — existing filter/set tests unaffected (null-list rules still normalize to wildcard; concrete lists unchanged). If a pre-existing set test used a null rule list and asserted wildcard behaviour, it still passes. - [ ] **Step 9: Ruff + commit** @@ -786,7 +860,7 @@ git commit -m "feat(accumulator): set coalescing/compatible/NA-flag on in-band s - Test: `tests/accumulator/test_engine.py` **Interfaces:** -- Consumes: `validate_set_columns`, `normalize_set_expr`, `set_wildcard_predicate` (Task 2); the accumulator compiler set branches (Task 4). +- Consumes: `validate_set_columns`, `validate_set_no_null_elements`, `normalize_set_expr`, `set_wildcard_predicate` (Task 2); the accumulator compiler set branches (Task 4). - Produces: `build(rules)` with any set dimension produces a lattice whose set `co_` columns are non-null and canonical; the frontier dedupes wildcard combinations. - [ ] **Step 1: Write the failing build regression tests** @@ -820,17 +894,19 @@ class TestSetMembershipBuildFrontier: assert sorted(by_pp[6]) == ["NZ", "UK"] def test_same_set_different_order_dedupes(self): - # Ordering: two rules whose sets are equal up to order must produce one combo. + # Ordering: two rules whose sets are equal up to order must dedupe to the + # single maximal combination — assert the EXACT surviving prime-product set. rules = pl.DataFrame({ "rule_name": ["R1", "R2"], "region": pl.Series("region", [["UK", "NZ"], ["NZ", "UK"]], dtype=pl.List(pl.Utf8)), }) engine = AccumulatorEngine(dimension_metadata=self._metadata()) rows = _rows(engine.build(rules).combinations) - # {R1,R2} intersection == {NZ,UK}; both singletons canonicalize equal, so - # the maximal pp=6 combination is present and carries the canonical set. + # R1 and R2 have equal (canonicalized) sets, are compatible (non-empty + # intersection), so {R1,R2} (pp=6) dominates both singletons {2},{3}. + assert set(rows["__prime_product"]) == {6} by_pp = dict(zip(rows["__prime_product"], rows["co_region"])) - assert by_pp[6] == ["NZ", "UK"] + assert by_pp[6] == ["NZ", "UK"] # canonical (sorted-unique) class TestSetExclusionBuild: @@ -870,7 +946,7 @@ class TestSetBuildValidation: - [ ] **Step 2: Run to verify it fails** Run: `hatch run test:test-target tests/accumulator/test_engine.py::TestSetMembershipBuildFrontier tests/accumulator/test_engine.py::TestSetExclusionBuild tests/accumulator/test_engine.py::TestSetBuildValidation -v` -Expected: FAIL — the accumulator does not yet normalize set columns; the compiler set branches exist (Task 4) but `co_region` starts null for wildcards, so `test_three_wildcard_rules_collapse_to_single_maximal` returns `{2,3,5,6,10,15,30}`, and `TestSetBuildValidation` does not raise. +Expected: FAIL — the accumulator does not yet normalize set columns. With Task 4's compiler but no normalization, wildcard `co_region` is null, so `set_wildcard_predicate` (a `list.contains` on a null list) yields **null**, compatibility becomes null (not true), and `_expand_level`'s filter drops those expansions — `test_three_wildcard_rules_collapse_to_single_maximal` returns **too few** combinations (e.g. only singletons `{2,3,5}`), which is `!= {30}`, so it fails. `test_two_membership_rules_coalesce_to_intersection` (concrete, non-null lists) may already pass — it is an integration check, not a fail-first unit. `TestSetBuildValidation` fails because no validation runs yet. (The point of the anchor test is the post-normalization assertion `== {30}` in Step 7, not the exact pre-normalization value.) - [ ] **Step 3: Add the normalization stage import** @@ -878,6 +954,7 @@ In `src/mountainash_rules/engines/accumulator/engine.py`, add near the other `mo ```python from mountainash_rules.core.set_wildcard import ( validate_set_columns, + validate_set_no_null_elements, normalize_set_expr, set_wildcard_predicate, ) @@ -901,7 +978,8 @@ Add this method to `AccumulatorEngine` (near `_create_anchor`): if not set_dims: return rules_pl rel = relation(rules_pl) - validate_set_columns(rel, set_dims) + validate_set_columns(rel, set_dims) # reservation (portable) + validate_set_no_null_elements(rel, set_dims) # element-nulls (polars build only) rel = rel.with_columns(*[ normalize_set_expr(dim, ma.col(dim.resolved_rule_field)).alias(dim.resolved_rule_field) for dim in set_dims @@ -963,7 +1041,7 @@ Add the assertion method: f"normalization did not reach every build path" ) ``` -> `checked[f"__null_{c}"]` indexes a collected native frame (polars/pandas); `sum(...)` over its values is backend-pure (builtin). This is a cheap invariant guard, not hot-path code. +> `checked[f"__null_{c}"]` indexes a collected native frame (polars/pandas); `sum(...)` over its values is backend-pure (builtin). This is a cheap invariant guard, not hot-path code. **Empty-build path:** `_normalize_set_columns` runs *before* the `n_rules == 0` early return (Step 4), so an empty build already carries normalized (0-row) set columns; the assertion is vacuously satisfied there and need not run on that path. Multi-level builds reach this assertion on `all_combos` (all levels concatenated), covering the "multi-level" case; partition-filtered builds reach it too (the filter happens before `to_polars`, upstream of normalization). - [ ] **Step 7: Run the build tests to verify they pass** @@ -986,7 +1064,7 @@ git commit -m "feat(accumulator): set-column normalization stage; frontier untou --- -### Task 6: Apply round-trip, float typing, idempotence, docs +### Task 6: Apply round-trip, float typing, docs (idempotence covered in Task 2) **Files:** - Test: `tests/accumulator/test_apply.py` (apply round-trip) @@ -997,9 +1075,9 @@ git commit -m "feat(accumulator): set-column normalization stage; frontier untou - Consumes: everything from Tasks 1–5. - Produces: end-to-end verification + docs. Nothing downstream. -- [ ] **Step 1: Write the failing apply round-trip test** +- [ ] **Step 1: Write the apply round-trip verification test** -Add to `tests/accumulator/test_apply.py` (uses `AccumulatorEngine`, `Dimension`, `DimensionsMetadata`, `_rows`, `pl` already present; add `from mountainash_rules.core.constants import DataType` if absent): +This is an integration **verification** test (expected to PASS on first run — Tasks 2–5 already implement build+apply), not a fail-first unit. Add to `tests/accumulator/test_apply.py` (uses `AccumulatorEngine`, `Dimension`, `DimensionsMetadata`, `_rows`, `pl` already present; add `from mountainash_rules.core.constants import DataType` if absent): ```python class TestSetMembershipApply: def _metadata(self): @@ -1072,11 +1150,11 @@ In `CLAUDE.md`, the Match Strategies table row for `set_membership` / `set_exclu ``` | `set_membership` / `set_exclusion` | list column | Polars-native fallback | ``` -Replace with: +Replace with (note: the filter path is backend-agnostic `t_is_in`/`t_is_not_in`, polars/ibis, with narwhals under `mountainash#89` — NOT "polars-native fallback"): ``` -| `set_membership` / `set_exclusion` | list column | Polars-native fallback; accumulator-coalesceable (membership → list intersection, exclusion → list union). Wildcard = in-band `[unknown_sentinel_for(dtype)]` (never null; bool unsupported); see `null-is-not-a-portable-sentinel` principle. | +| `set_membership` / `set_exclusion` | list column | Filter via `t_is_in`/`t_is_not_in` (polars/ibis; narwhals list ops under `mountainash#89`). Accumulator-coalesceable (membership → list intersection, exclusion → list union). Wildcard = in-band `[unknown_sentinel_for(dtype)]` (never null; bool unsupported); see `null-is-not-a-portable-sentinel` principle. | ``` -And under the "Ternary match logic" / sentinels area, add one line noting that set-dimension wildcards use the in-band `[sentinel]` list (not null), normalized at ingestion in both engines. +And under the "Ternary match logic" / sentinels area, add one line noting that set-dimension wildcards use the in-band `[sentinel]` list (not null), normalized at ingestion (reservation check in both engines; element-null check in the accumulator build). - [ ] **Step 6: Full quick suite + purity** diff --git a/docs/superpowers/specs/2026-07-20-accumulator-set-sentinel-wildcard-design.md b/docs/superpowers/specs/2026-07-20-accumulator-set-sentinel-wildcard-design.md index c5b1dd5..bfa8817 100644 --- a/docs/superpowers/specs/2026-07-20-accumulator-set-sentinel-wildcard-design.md +++ b/docs/superpowers/specs/2026-07-20-accumulator-set-sentinel-wildcard-design.md @@ -96,7 +96,12 @@ def set_wildcard_predicate(dim, col): return col.list.contains(ma.lit(_typed_sentinel(dim))) # typed scalar sentinel ``` -**Ingestion validation (guards the reservation, F2/F6/F7).** A separate validation pass over the materialized rule frame — run once at ingestion in *both* engines — raises `ValueError` when, for any set dimension, a **non-null** rule list either (a) contains the sentinel but is not exactly `[sentinel]` (mixed/embedded sentinel), or (b) contains a null element. This makes "reserved, out-of-domain" an enforced contract, not an assumption, so `set_wildcard_predicate` is unambiguous. Whole-list null is *not* an error — it is the valid wildcard, normalized to `[sentinel]`. +**Ingestion validation (guards the reservation, F2/F6/F7).** A validation pass over the materialized rule frame raises `ValueError` on invalid set rule lists, split by portability: + +- **Reservation check (portable — both engines, all backends):** a **non-null** rule list that contains the sentinel but is not exactly `[sentinel]` (mixed/embedded sentinel) is rejected. Uses only `list.contains` + `list.len`, both Ibis/Narwhals-safe. This makes "reserved, out-of-domain" an enforced contract, so `set_wildcard_predicate` is unambiguous. +- **Null-element check (accumulator build only):** a non-null rule list containing a null *element* is rejected. This needs `list.drop_nulls`, which mountainash's **Ibis backend does not support** — so it runs only in the accumulator build (polars-internal, where the op is available), where element-nulls actually corrupt sort/unique/set-ops. The filter engine does not run it: `t_is_in` tolerates a null element benignly (it simply matches nothing), so a standalone filter engine over Ibis set rules is unaffected. + +Whole-list null is *not* an error in either check — it is the valid wildcard, normalized to `[sentinel]`. **Normalization sequencing:** validation → `normalize_set_expr` MUST run at ingestion, before any detection/coalesce/fingerprint. This is what makes every downstream predicate see only non-null, validated lists. @@ -132,7 +137,7 @@ The fix is in the **data**, not the join — so `_frontier_filter` is **untouche - both-concrete → `canonicalize_set_expr(set_intersection|set_union)` so equal coalesced sets share a fingerprint. Membership uses `set_intersection`; exclusion uses `set_union`. (Wildcard is the identity for both: `∩` with match-anything, `∪` with exclude-nothing.) 4. **`_compatible_set_membership`:** `co_wild OR rhs_wild OR (intersection nonempty)` with `set_wildcard_predicate` for the wildcard tests; concrete∩concrete-nonempty unchanged. `SET_EXCLUSION` remains always-compatible (`ma.lit(True)`). -5. **`co__na` flag = `set_wildcard_predicate` applied to the FINAL coalesced value (F8)**, not to the LHS/RHS inputs. So wildcard+concrete → concrete coalesced value → flag **false**; wildcard+wildcard → `[sentinel]` → flag **true**; concrete+concrete → concrete → false. The flag tracks whether the *combination* leaves the dimension unconstrained, which is the semantics the frontier fingerprint and apply/provenance need. +5. **`co__na` flag = whether the FINAL coalesced value is a wildcard (F8)**, computed as the equivalent input formula **`set_wildcard_predicate(co) AND set_wildcard_predicate(rhs)`**. Under the validated, normalized representation the coalesce yields `[sentinel]` **iff both inputs are wildcard** (wildcard+concrete → the concrete side; concrete+concrete → intersection/union), so `co_wild ∧ rhs_wild` is exactly the wildcard status of the coalesced output — and, unlike a literal read of the output, it is computable from the LHS/RHS inputs in the *same* `with_columns` pass (a column cannot reference a sibling being computed in the same pass). So: wildcard+concrete → **false**; wildcard+wildcard → **true**; concrete+concrete → **false**. The flag tracks whether the *combination* leaves the dimension unconstrained, which is what the frontier fingerprint and apply/provenance need. 6. **`_frontier_filter`: unchanged.** Inputs are now null-free and canonical, so the self-join dedupes wildcard combinations, and equal-but-differently-ordered sets dedupe too. **Apply consistency (free).** After build, the lattice's `co_` columns *are* the normalized representation. Apply runs the filter compiler (above) over them; `set_wildcard_predicate` short-circuits identically. `normalize_set_expr` is **idempotent** — re-applying it to already-normalized `co_` columns is a no-op (the `is_null` branch never fires; canonicalizing an already-canonical list is stable) — so the filter compiler's inline normalization is safe over lattice columns. One representation, both engines, build → apply. From 13841074434a2eaf8b9d18038cd057bedcec0269 Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 20 Jul 2026 13:26:11 +1000 Subject: [PATCH 05/10] feat(dimension): reject bool set dimensions (no typed wildcard sentinel) Co-Authored-By: Claude Opus 4.8 (1M context) --- src/mountainash_rules/core/dimension.py | 12 ++++++++++ tests/core/test_dimension.py | 30 +++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/src/mountainash_rules/core/dimension.py b/src/mountainash_rules/core/dimension.py index 336577d..e481973 100644 --- a/src/mountainash_rules/core/dimension.py +++ b/src/mountainash_rules/core/dimension.py @@ -131,6 +131,18 @@ def _validate_strategy_fields(self) -> "Dimension": f"temporal type" ) + if self.match_strategy in ( + MatchStrategy.SET_MEMBERSHIP, + MatchStrategy.SET_EXCLUSION, + ): + if self.data_type is DataType.BOOL: + raise ValueError( + f"Dimension '{self.dimension_name}' uses " + f"{self.match_strategy.value} with data_type bool; boolean " + f"set dimensions are not supported (no typed wildcard sentinel " + f"exists and a set over {{true, false}} is degenerate)" + ) + return self diff --git a/tests/core/test_dimension.py b/tests/core/test_dimension.py index 52c7772..a239414 100644 --- a/tests/core/test_dimension.py +++ b/tests/core/test_dimension.py @@ -190,3 +190,33 @@ def test_existing_dimensions_unaffected(self): ), ]) assert all(d.role == DimensionRole.CONSTRAINT for d in metadata.dimensions) + + +class TestSetDimensionBoolRejected: + def test_bool_set_membership_rejected(self): + import pytest + from pydantic import ValidationError + from mountainash_rules import Dimension + from mountainash_rules.core.constants import MatchStrategy, DataType + with pytest.raises(ValidationError, match="bool"): + Dimension(dimension_name="flags", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.BOOL) + + def test_bool_set_exclusion_rejected(self): + import pytest + from pydantic import ValidationError + from mountainash_rules import Dimension + from mountainash_rules.core.constants import MatchStrategy, DataType + with pytest.raises(ValidationError, match="bool"): + Dimension(dimension_name="flags", match_strategy=MatchStrategy.SET_EXCLUSION, data_type=DataType.BOOL) + + def test_str_set_membership_allowed(self): + from mountainash_rules import Dimension + from mountainash_rules.core.constants import MatchStrategy, DataType + d = Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.STR) + assert d.data_type is DataType.STR + + def test_int_set_membership_allowed(self): + from mountainash_rules import Dimension + from mountainash_rules.core.constants import MatchStrategy, DataType + d = Dimension(dimension_name="tiers", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.INT) + assert d.data_type is DataType.INT From e240ead1007d894ad52211c9da8fe9ec5d0d96a2 Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 20 Jul 2026 13:53:16 +1000 Subject: [PATCH 06/10] feat(core): shared in-band sentinel set-wildcard helpers + validation Includes cross-assertions pinning each validator to its own violation class (reservation vs null-element) and a direct sentinel_list_expr typing test, per sol task review. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/mountainash_rules/core/set_wildcard.py | 121 +++++++++++++++++ tests/core/test_set_wildcard.py | 151 +++++++++++++++++++++ 2 files changed, 272 insertions(+) create mode 100644 src/mountainash_rules/core/set_wildcard.py create mode 100644 tests/core/test_set_wildcard.py diff --git a/src/mountainash_rules/core/set_wildcard.py b/src/mountainash_rules/core/set_wildcard.py new file mode 100644 index 0000000..5ad3639 --- /dev/null +++ b/src/mountainash_rules/core/set_wildcard.py @@ -0,0 +1,121 @@ +"""Shared in-band sentinel representation for set-dimension wildcards. + +A SET_MEMBERSHIP/SET_EXCLUSION wildcard is the single-element list +``[unknown_sentinel_for(dim.data_type)]`` — never a null. These helpers are the +single source of truth for both the filter engine (``core/compiler.py``) and the +accumulator engine (``engines/accumulator/``), so the two cannot diverge. + +The sentinel is a reserved, out-of-domain value; ``validate_set_columns`` +enforces that a concrete rule list never embeds it, so ``set_wildcard_predicate`` +is unambiguous. +""" + +from __future__ import annotations + +import typing as t + +import mountainash.expressions as ma +from mountainash.expressions import BaseExpressionAPI + +from mountainash_rules.core.constants import DataType, unknown_sentinel_for +from mountainash_rules.core.dimension import Dimension + + +def _typed_sentinel(dim: Dimension) -> t.Any: + """The dimension's wildcard sentinel, coerced to the dim's Python element type.""" + sent = unknown_sentinel_for(dim.data_type) + return float(sent) if dim.data_type == DataType.FLOAT else sent + + +def sentinel_list_expr(dim: Dimension) -> BaseExpressionAPI: + """A ``[sentinel]`` list literal whose element carries the dim's Python type. + + Element coercion (not a native list-dtype cast) keeps this backend-pure: a + Python-float element yields a ``List(Float64)`` literal with no cast. + """ + return ma.lit([_typed_sentinel(dim)]) + + +def canonicalize_set_expr(col: BaseExpressionAPI) -> BaseExpressionAPI: + """Sort + dedupe a list column so equal sets compare/fingerprint identically.""" + return col.list.unique().list.sort() + + +def normalize_set_expr(dim: Dimension, col: BaseExpressionAPI) -> BaseExpressionAPI: + """Null list -> ``[sentinel]``; concrete list -> sorted-unique. The one normaliser. + + Idempotent: re-applying to an already-normalized column is a no-op. + """ + return ma.when(col.is_null()).then(sentinel_list_expr(dim)).otherwise( + canonicalize_set_expr(col) + ) + + +def set_wildcard_predicate(dim: Dimension, col: BaseExpressionAPI) -> BaseExpressionAPI: + """True when ``col`` (post-normalization) is the wildcard. + + Because ``validate_set_columns`` rejects any concrete list embedding the + sentinel, ``list.contains(sentinel)`` is true iff the list is exactly + ``[sentinel]``. Must be evaluated after normalization (``contains`` returns + null on a null cell). + """ + return col.list.contains(ma.lit(_typed_sentinel(dim))) + + +def _embedded_sentinel_predicate(dim: Dimension, field: str) -> BaseExpressionAPI: + """True for a non-null list that contains the sentinel but is not ``[sentinel]``.""" + col = ma.col(field) + has_sentinel = col.list.contains(ma.lit(_typed_sentinel(dim))) + length = col.list.len() + return col.is_not_null().__and__(has_sentinel.__and__(length.ne(ma.lit(1)))) + + +def _null_element_predicate(field: str) -> BaseExpressionAPI: + """True for a non-null list that contains a null element. + + Uses ``list.drop_nulls`` — mountainash's IBIS backend raises + ``BackendCapabilityError`` for this op, so callers must only run this on the + polars-internal accumulator build path (see ``validate_set_no_null_elements``). + """ + col = ma.col(field) + length = col.list.len() + non_null_length = col.list.drop_nulls().list.len() + return col.is_not_null().__and__(length.ne(non_null_length)) + + +def validate_set_columns(rules_rel: t.Any, set_dims: list[Dimension]) -> None: + """Raise ``ValueError`` if a concrete set-rule list embeds the reserved sentinel. + + Portable — uses only ``list.contains`` + ``list.len`` (Ibis/Narwhals-safe), so + it runs in BOTH engines on any backend. A whole-list null is valid (the + wildcard). ``rules_rel`` is a ``mountainash`` relation; ``collect`` returns a + native frame that supports builtin ``len`` (backend-pure — no native import). + """ + for dim in set_dims: + field = dim.resolved_rule_field + embedded = rules_rel.filter(_embedded_sentinel_predicate(dim, field)).collect() + if len(embedded) > 0: + raise ValueError( + f"Dimension '{dim.dimension_name}': a concrete rule list embeds the " + f"reserved wildcard sentinel {unknown_sentinel_for(dim.data_type)!r}. " + f"The sentinel is only valid as the sole element (the wildcard)." + ) + + +def validate_set_no_null_elements(rules_rel: t.Any, set_dims: list[Dimension]) -> None: + """Raise ``ValueError`` if a set-rule list contains a null element. + + Uses ``list.drop_nulls`` (Ibis-unsupported), so this is called ONLY on the + polars-internal accumulator build path. The filter engine does not call it: + ``t_is_in`` tolerates a null element (it matches nothing), so a standalone + filter engine over Ibis set rules is unaffected. + """ + for dim in set_dims: + field = dim.resolved_rule_field + null_elem = rules_rel.filter(_null_element_predicate(field)).collect() + if len(null_elem) > 0: + raise ValueError( + f"Dimension '{dim.dimension_name}': a rule list contains a null " + f"element. Element-level nulls are not allowed; use a whole-list " + f"null (or omit the cell) for a wildcard." + ) diff --git a/tests/core/test_set_wildcard.py b/tests/core/test_set_wildcard.py new file mode 100644 index 0000000..00777bd --- /dev/null +++ b/tests/core/test_set_wildcard.py @@ -0,0 +1,151 @@ +"""Tests for the shared set-wildcard sentinel helpers.""" + +import polars as pl +import pytest + +from mountainash_rules.core.constants import MatchStrategy, DataType +from mountainash_rules.core.dimension import Dimension +from mountainash_rules.core.set_wildcard import ( + sentinel_list_expr, + canonicalize_set_expr, + normalize_set_expr, + set_wildcard_predicate, + validate_set_columns, + validate_set_no_null_elements, +) +from mountainash.relations import relation + + +def _str_dim(): + return Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.STR) + + +def _float_dim(): + return Dimension(dimension_name="scores", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.FLOAT) + + +class TestNormalizeAndDetect: + def test_null_becomes_sentinel_list(self): + dim = _str_dim() + df = pl.DataFrame({"region": pl.Series("region", [None, ["AU"]], dtype=pl.List(pl.Utf8))}) + out = df.with_columns(normalize_set_expr(dim, __import__("mountainash").col("region")).alias("n").compile(df, booleanizer=None)) + assert out["n"].to_list() == [[""], ["AU"]] + assert out["n"].dtype == pl.List(pl.Utf8) + + def test_concrete_list_sorted_and_deduped(self): + dim = _str_dim() + import mountainash as ma + df = pl.DataFrame({"region": pl.Series("region", [["UK", "NZ", "NZ"]], dtype=pl.List(pl.Utf8))}) + out = df.with_columns(normalize_set_expr(dim, ma.col("region")).alias("n").compile(df, booleanizer=None)) + assert out["n"].to_list() == [["NZ", "UK"]] + + def test_wildcard_predicate_true_on_sentinel_false_on_concrete(self): + dim = _str_dim() + import mountainash as ma + df = pl.DataFrame({"region": pl.Series("region", [[""], ["AU"], []], dtype=pl.List(pl.Utf8))}) + out = df.with_columns(set_wildcard_predicate(dim, ma.col("region")).alias("w").compile(df, booleanizer=None)) + assert out["w"].to_list() == [True, False, False] + + def test_float_dim_sentinel_list_is_float_typed(self): + dim = _float_dim() + import mountainash as ma + df = pl.DataFrame({"scores": pl.Series("scores", [None], dtype=pl.List(pl.Float64))}) + out = df.with_columns(normalize_set_expr(dim, ma.col("scores")).alias("n").compile(df, booleanizer=None)) + assert out["n"].dtype == pl.List(pl.Float64) + assert out["n"].to_list() == [[-999999999.0]] + + def test_normalize_idempotent(self): + dim = _str_dim() + import mountainash as ma + df = pl.DataFrame({"region": pl.Series("region", [None, ["UK", "NZ"]], dtype=pl.List(pl.Utf8))}) + once = df.with_columns(normalize_set_expr(dim, ma.col("region")).alias("region").compile(df, booleanizer=None)) + twice = once.with_columns(normalize_set_expr(dim, ma.col("region")).alias("region").compile(once, booleanizer=None)) + assert once["region"].to_list() == twice["region"].to_list() + + def test_normalize_idempotent_float_and_date(self): + # F9: idempotence must hold across dtypes, not just str. + import datetime as _dt + import mountainash as ma + from mountainash_rules.core.constants import DataType + float_dim = _float_dim() + fdf = pl.DataFrame({"scores": pl.Series("scores", [None, [2.5, 1.5]], dtype=pl.List(pl.Float64))}) + f1 = fdf.with_columns(normalize_set_expr(float_dim, ma.col("scores")).alias("scores").compile(fdf, booleanizer=None)) + f2 = f1.with_columns(normalize_set_expr(float_dim, ma.col("scores")).alias("scores").compile(f1, booleanizer=None)) + assert f1["scores"].to_list() == f2["scores"].to_list() + assert f1["scores"].to_list() == [[-999999999.0], [1.5, 2.5]] + + date_dim = Dimension(dimension_name="days", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.DATE) + d = [_dt.date(2020, 1, 2), _dt.date(2020, 1, 1)] + ddf = pl.DataFrame({"days": pl.Series("days", [None, d], dtype=pl.List(pl.Date))}) + d1 = ddf.with_columns(normalize_set_expr(date_dim, ma.col("days")).alias("days").compile(ddf, booleanizer=None)) + d2 = d1.with_columns(normalize_set_expr(date_dim, ma.col("days")).alias("days").compile(d1, booleanizer=None)) + assert d1["days"].to_list() == d2["days"].to_list() + assert d1["days"].dtype == pl.List(pl.Date) + + +class TestCanonicalize: + def test_sort_and_dedupe(self): + import mountainash as ma + df = pl.DataFrame({"c": pl.Series("c", [["UK", "NZ", "UK"]], dtype=pl.List(pl.Utf8))}) + out = df.with_columns(canonicalize_set_expr(ma.col("c")).alias("c2").compile(df, booleanizer=None)) + assert out["c2"].to_list() == [["NZ", "UK"]] + + +class TestValidateSetColumns: + def test_embedded_sentinel_rejected(self): + dim = _str_dim() + rules = pl.DataFrame({"region": pl.Series("region", [["AU", ""]], dtype=pl.List(pl.Utf8))}) + with pytest.raises(ValueError, match="sentinel"): + validate_set_columns(relation(rules), [dim]) + + def test_embedded_sentinel_passes_null_element_check(self): + # validate_set_columns is reservation-only; whole-list null + concrete OK. + dim = _str_dim() + rules = pl.DataFrame({"region": pl.Series("region", [None, ["AU"], [""]], dtype=pl.List(pl.Utf8))}) + validate_set_columns(relation(rules), [dim]) # no raise + + def test_null_element_rejected_by_dedicated_check(self): + dim = _str_dim() + rules = pl.DataFrame({"region": pl.Series("region", [["AU", None]], dtype=pl.List(pl.Utf8))}) + with pytest.raises(ValueError, match="null element"): + validate_set_no_null_elements(relation(rules), [dim]) + + def test_null_element_check_allows_whole_list_null(self): + dim = _str_dim() + rules = pl.DataFrame({"region": pl.Series("region", [None, ["AU"]], dtype=pl.List(pl.Utf8))}) + validate_set_no_null_elements(relation(rules), [dim]) # no raise + + def test_no_set_dims_is_noop(self): + validate_set_columns(relation(pl.DataFrame({"x": [1]})), []) + validate_set_no_null_elements(relation(pl.DataFrame({"x": [1]})), []) + + def test_reservation_check_ignores_null_elements(self): + # The split is directional: validate_set_columns is reservation-only and + # must NOT reject an element-null list (that is the other validator's job). + dim = _str_dim() + rules = pl.DataFrame({"region": pl.Series("region", [["AU", None]], dtype=pl.List(pl.Utf8))}) + validate_set_columns(relation(rules), [dim]) # no raise + + def test_null_element_check_ignores_embedded_sentinel(self): + # ...and validate_set_no_null_elements must NOT reject an embedded-sentinel + # list (that is the reservation check's job). Together with the two tests + # above, this pins each validator to exactly its own violation class. + dim = _str_dim() + rules = pl.DataFrame({"region": pl.Series("region", [["AU", ""]], dtype=pl.List(pl.Utf8))}) + validate_set_no_null_elements(relation(rules), [dim]) # no raise + + +class TestSentinelListExpr: + def test_str_sentinel_list(self): + dim = _str_dim() + df = pl.DataFrame({"x": [0]}) + out = df.with_columns(sentinel_list_expr(dim).alias("s").compile(df, booleanizer=None)) + assert out["s"].to_list() == [[""]] + assert out["s"].dtype == pl.List(pl.Utf8) + + def test_float_sentinel_list_is_float_typed(self): + dim = _float_dim() + df = pl.DataFrame({"x": [0]}) + out = df.with_columns(sentinel_list_expr(dim).alias("s").compile(df, booleanizer=None)) + assert out["s"].to_list() == [[-999999999.0]] + assert out["s"].dtype == pl.List(pl.Float64) From 833d7f7064d4e8c01886a50eda72f11514c974aa Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 20 Jul 2026 14:08:22 +1000 Subject: [PATCH 07/10] feat(filter): set-wildcard ternary short-circuit + reject invalid set rule lists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the in-band sentinel into the filter compiler (normalize rule list, short-circuit wildcard to ternary 0) and validate set columns once on both scoring entry points (single via _scored_relation, batch via _evaluate_batch_frame). Also fix set_wildcard validators to count rows with the relation's portable count_rows() instead of len(collect()) — the latter raises on the Ibis backend (ExpressionError: Use .count() instead), which broke the ibis-duckdb mixed-strategy integration tests once the validator was first wired in. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/mountainash_rules/core/compiler.py | 21 ++--- src/mountainash_rules/core/set_wildcard.py | 13 +-- .../engines/filter/engine.py | 22 +++++ tests/core/test_compiler.py | 90 +++++++++++++++++++ tests/filter/test_engine.py | 34 +++++++ 5 files changed, 164 insertions(+), 16 deletions(-) diff --git a/src/mountainash_rules/core/compiler.py b/src/mountainash_rules/core/compiler.py index e752948..676150f 100644 --- a/src/mountainash_rules/core/compiler.py +++ b/src/mountainash_rules/core/compiler.py @@ -15,6 +15,7 @@ unknown_sentinel_for, ) from mountainash_rules.core.dimension import Dimension, DimensionsMetadata +from mountainash_rules.core.set_wildcard import normalize_set_expr, set_wildcard_predicate class DimensionCompiler: @@ -208,15 +209,15 @@ def _compile_context_regex(self, dim: Dimension) -> BaseExpressionAPI: return ma.when(match).then(1).otherwise(-1) def _compile_set_membership(self, dim: Dimension) -> BaseExpressionAPI: - """SET_MEMBERSHIP: context value is in the rule's list column.""" - sentinels = sentinels_for(dim.data_type) - rule_col = ma.col(dim.resolved_rule_field) - ctx_col = ma.t_col(CTX_PREFIX + dim.dimension_name, unknown=sentinels) - return ctx_col.t_is_in(rule_col) + """SET_MEMBERSHIP: context value in the rule list; wildcard rule -> ternary 0.""" + rule_col = normalize_set_expr(dim, ma.col(dim.resolved_rule_field)) + ctx_col = ma.t_col(CTX_PREFIX + dim.dimension_name, unknown=sentinels_for(dim.data_type)) + is_wild = set_wildcard_predicate(dim, rule_col) + return ma.when(is_wild).then(0).otherwise(ctx_col.t_is_in(rule_col)) def _compile_set_exclusion(self, dim: Dimension) -> BaseExpressionAPI: - """SET_EXCLUSION: context value is NOT in the rule's list column.""" - sentinels = sentinels_for(dim.data_type) - rule_col = ma.col(dim.resolved_rule_field) - ctx_col = ma.t_col(CTX_PREFIX + dim.dimension_name, unknown=sentinels) - return ctx_col.t_is_not_in(rule_col) + """SET_EXCLUSION: context value NOT in the rule list; wildcard rule -> ternary 0.""" + rule_col = normalize_set_expr(dim, ma.col(dim.resolved_rule_field)) + ctx_col = ma.t_col(CTX_PREFIX + dim.dimension_name, unknown=sentinels_for(dim.data_type)) + is_wild = set_wildcard_predicate(dim, rule_col) + return ma.when(is_wild).then(0).otherwise(ctx_col.t_is_not_in(rule_col)) diff --git a/src/mountainash_rules/core/set_wildcard.py b/src/mountainash_rules/core/set_wildcard.py index 5ad3639..54b1ee8 100644 --- a/src/mountainash_rules/core/set_wildcard.py +++ b/src/mountainash_rules/core/set_wildcard.py @@ -88,13 +88,14 @@ def validate_set_columns(rules_rel: t.Any, set_dims: list[Dimension]) -> None: Portable — uses only ``list.contains`` + ``list.len`` (Ibis/Narwhals-safe), so it runs in BOTH engines on any backend. A whole-list null is valid (the - wildcard). ``rules_rel`` is a ``mountainash`` relation; ``collect`` returns a - native frame that supports builtin ``len`` (backend-pure — no native import). + wildcard). ``rules_rel`` is a ``mountainash`` relation; row existence is tested + with the relation's portable ``count_rows`` (backend-pure — no native import; + builtin ``len`` on a collected frame is NOT portable — Ibis rejects it). """ for dim in set_dims: field = dim.resolved_rule_field - embedded = rules_rel.filter(_embedded_sentinel_predicate(dim, field)).collect() - if len(embedded) > 0: + n_embedded = rules_rel.filter(_embedded_sentinel_predicate(dim, field)).count_rows() + if n_embedded > 0: raise ValueError( f"Dimension '{dim.dimension_name}': a concrete rule list embeds the " f"reserved wildcard sentinel {unknown_sentinel_for(dim.data_type)!r}. " @@ -112,8 +113,8 @@ def validate_set_no_null_elements(rules_rel: t.Any, set_dims: list[Dimension]) - """ for dim in set_dims: field = dim.resolved_rule_field - null_elem = rules_rel.filter(_null_element_predicate(field)).collect() - if len(null_elem) > 0: + n_null_elem = rules_rel.filter(_null_element_predicate(field)).count_rows() + if n_null_elem > 0: raise ValueError( f"Dimension '{dim.dimension_name}': a rule list contains a null " f"element. Element-level nulls are not allowed; use a whole-list " diff --git a/src/mountainash_rules/engines/filter/engine.py b/src/mountainash_rules/engines/filter/engine.py index f9034b8..8346427 100644 --- a/src/mountainash_rules/engines/filter/engine.py +++ b/src/mountainash_rules/engines/filter/engine.py @@ -18,6 +18,7 @@ CTX_PREFIX, NOT_SET, HitPolicy, + MatchStrategy, not_set_sentinel_for, ) from mountainash_rules.core.context import extract_context_values @@ -33,6 +34,7 @@ selection_info_from_metadata, ) from mountainash_rules.core.result import ExplainResult, RuleResult +from mountainash_rules.core.set_wildcard import validate_set_columns class ExpressionRulesEngine: @@ -72,6 +74,24 @@ def __init__( self._metadata = None self._rules = rules + self._set_dims_validated = False + + def _validate_set_rules_once(self) -> None: + """Reject set rule lists that embed the reserved sentinel — once, portably. + + Called from BOTH scoring entry points (single and batch) because + evaluate_batch does not route through _scored_relation. metadata is None + when the engine was built from dimension_expressions (no Dimension objects + to inspect), so skip that case. + """ + if self._metadata is None or self._set_dims_validated: + return + set_dims = [ + d for d in self._metadata.dimensions + if d.match_strategy in (MatchStrategy.SET_MEMBERSHIP, MatchStrategy.SET_EXCLUSION) + ] + validate_set_columns(relation(self._rules), set_dims) + self._set_dims_validated = True def evaluate( self, @@ -310,6 +330,7 @@ def _evaluate_batch_frame( min_specificity: int | None, include_observability: bool, ) -> t.Any: + self._validate_set_rules_once() rules_rel = relation(self._rules) self._check_reserved(rules_rel, "Rules") rules_rel = rules_rel.with_row_index(name="__rule_index") @@ -406,6 +427,7 @@ def _evaluate_batch_frame( def _scored_relation(self, active_dims: list[str], context_values: dict[str, t.Any]) -> t.Any: """Rules frame scored against a context: ternaries + __survived + __specificity, unfiltered.""" + self._validate_set_rules_once() # Steps 0-3: reserved-column guard + row index, bind ctx literals, # ternary columns, survival + specificity. Steps 4-7 (filter, rank, # assertions, cardinality, drop) stay in the callers. diff --git a/tests/core/test_compiler.py b/tests/core/test_compiler.py index d07ec70..8734042 100644 --- a/tests/core/test_compiler.py +++ b/tests/core/test_compiler.py @@ -724,3 +724,93 @@ def test_bool_rule_null_is_wildcard_context_null_is_not(self, compiler): result = df.with_columns(expr.name.alias("__t_flag").compile(df, booleanizer=None)) # rule null -> 0 regardless of context; context null vs specific -> -1 assert result["__t_flag"].to_list() == [1, 0, -1, 0] + + +class TestSetMembershipTernary: + def _compile(self, dim): + from mountainash_rules.core.compiler import DimensionCompiler + return DimensionCompiler().compile_dimension(dim) + + def _dim(self): + from mountainash_rules.core.constants import DataType + return Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.STR) + + def test_wildcard_rule_is_ternary_zero(self): + from mountainash_rules.core.constants import CTX_PREFIX + expr = self._compile(self._dim()) + df = pl.DataFrame({ + "region": pl.Series("region", [[""]], dtype=pl.List(pl.Utf8)), + CTX_PREFIX + "region": ["AU"], + }) + out = df.with_columns(expr.alias("t").compile(df, booleanizer=None)) + assert out["t"].to_list() == [0] + + def test_context_in_set_is_one(self): + from mountainash_rules.core.constants import CTX_PREFIX + expr = self._compile(self._dim()) + df = pl.DataFrame({ + "region": pl.Series("region", [["AU", "NZ"]], dtype=pl.List(pl.Utf8)), + CTX_PREFIX + "region": ["AU"], + }) + out = df.with_columns(expr.alias("t").compile(df, booleanizer=None)) + assert out["t"].to_list() == [1] + + def test_context_out_of_set_is_minus_one(self): + from mountainash_rules.core.constants import CTX_PREFIX + expr = self._compile(self._dim()) + df = pl.DataFrame({ + "region": pl.Series("region", [["AU", "NZ"]], dtype=pl.List(pl.Utf8)), + CTX_PREFIX + "region": ["US"], + }) + out = df.with_columns(expr.alias("t").compile(df, booleanizer=None)) + assert out["t"].to_list() == [-1] + + def test_null_rule_list_normalizes_to_wildcard(self): + from mountainash_rules.core.constants import CTX_PREFIX + expr = self._compile(self._dim()) + df = pl.DataFrame({ + "region": pl.Series("region", [None], dtype=pl.List(pl.Utf8)), + CTX_PREFIX + "region": ["AU"], + }) + out = df.with_columns(expr.alias("t").compile(df, booleanizer=None)) + assert out["t"].to_list() == [0] + + +class TestSetExclusionTernary: + def _compile(self, dim): + from mountainash_rules.core.compiler import DimensionCompiler + return DimensionCompiler().compile_dimension(dim) + + def _dim(self): + from mountainash_rules.core.constants import DataType + return Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_EXCLUSION, data_type=DataType.STR) + + def test_wildcard_rule_is_zero(self): + from mountainash_rules.core.constants import CTX_PREFIX + expr = self._compile(self._dim()) + df = pl.DataFrame({ + "region": pl.Series("region", [[""]], dtype=pl.List(pl.Utf8)), + CTX_PREFIX + "region": ["AU"], + }) + out = df.with_columns(expr.alias("t").compile(df, booleanizer=None)) + assert out["t"].to_list() == [0] + + def test_context_in_excluded_set_is_minus_one(self): + from mountainash_rules.core.constants import CTX_PREFIX + expr = self._compile(self._dim()) + df = pl.DataFrame({ + "region": pl.Series("region", [["AU"]], dtype=pl.List(pl.Utf8)), + CTX_PREFIX + "region": ["AU"], + }) + out = df.with_columns(expr.alias("t").compile(df, booleanizer=None)) + assert out["t"].to_list() == [-1] + + def test_context_not_in_excluded_set_is_one(self): + from mountainash_rules.core.constants import CTX_PREFIX + expr = self._compile(self._dim()) + df = pl.DataFrame({ + "region": pl.Series("region", [["AU"]], dtype=pl.List(pl.Utf8)), + CTX_PREFIX + "region": ["NZ"], + }) + out = df.with_columns(expr.alias("t").compile(df, booleanizer=None)) + assert out["t"].to_list() == [1] diff --git a/tests/filter/test_engine.py b/tests/filter/test_engine.py index 32a64a5..427627c 100644 --- a/tests/filter/test_engine.py +++ b/tests/filter/test_engine.py @@ -210,3 +210,37 @@ def test_must_provide_one_of_metadata_or_expressions(self, backend_name): rules = build_backend_df(backend_name, {"rule_name": ["r1"]}, table_name="neither") with pytest.raises(ValueError, match="Must provide"): ExpressionRulesEngine(rules=rules) + + +class TestFilterEngineRejectsInvalidSetRules: + def _meta(self): + from mountainash_rules import Dimension, DimensionsMetadata + from mountainash_rules.core.constants import MatchStrategy, DataType + return DimensionsMetadata(dimensions=[ + Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.STR), + ]) + + def _rules(self): + import polars as pl + return pl.DataFrame({ + "rule_name": ["R1"], + "region": pl.Series("region", [["AU", ""]], dtype=pl.List(pl.Utf8)), + }) + + def test_embedded_sentinel_rejected_on_evaluate(self): + import pytest + from mountainash_rules import ExpressionRulesEngine + engine = ExpressionRulesEngine(rules=self._rules(), dimension_metadata=self._meta()) + with pytest.raises(ValueError, match="sentinel"): + engine.evaluate({"region": "AU"}) + + def test_embedded_sentinel_rejected_on_evaluate_batch(self): + # evaluate_batch does NOT route through _scored_relation — this covers the + # batch path explicitly (a batch-first call must still validate). + import polars as pl + import pytest + from mountainash_rules import ExpressionRulesEngine + engine = ExpressionRulesEngine(rules=self._rules(), dimension_metadata=self._meta()) + contexts = pl.DataFrame({"region": ["AU"]}) + with pytest.raises(ValueError, match="sentinel"): + engine.evaluate_batch(contexts) From 33cc65f87922283ede84ff3c24022fc83d81beaa Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 20 Jul 2026 14:12:25 +1000 Subject: [PATCH 08/10] feat(accumulator): set coalescing/compatible/NA-flag on in-band sentinel Co-Authored-By: Claude Opus 4.8 (1M context) --- .../engines/accumulator/compiler.py | 53 +++++++++- tests/accumulator/test_compiler.py | 96 +++++++++++++++++++ 2 files changed, 148 insertions(+), 1 deletion(-) diff --git a/src/mountainash_rules/engines/accumulator/compiler.py b/src/mountainash_rules/engines/accumulator/compiler.py index fb0756b..cb9bd63 100644 --- a/src/mountainash_rules/engines/accumulator/compiler.py +++ b/src/mountainash_rules/engines/accumulator/compiler.py @@ -10,6 +10,11 @@ unknown_sentinel_for, ) from mountainash_rules.core.dimension import Dimension +from mountainash_rules.core.set_wildcard import ( + set_wildcard_predicate, + canonicalize_set_expr, + sentinel_list_expr, +) class AccumulatorCompiler: @@ -34,6 +39,10 @@ def compile_compatible(self, dim: Dimension) -> BaseExpressionAPI: return self._compatible_range(dim) case MatchStrategy.GREATER_THAN | MatchStrategy.LESS_THAN: return self._compatible_threshold(dim) + case MatchStrategy.SET_MEMBERSHIP: + return self._compatible_set_membership(dim) + case MatchStrategy.SET_EXCLUSION: + return ma.lit(True) case _: raise ValueError( f"Strategy {dim.match_strategy.name} not supported by accumulator" @@ -50,13 +59,17 @@ def compile_coalesce(self, dim: Dimension) -> list[BaseExpressionAPI]: return self._coalesce_threshold(dim, ma.greatest) case MatchStrategy.LESS_THAN: return self._coalesce_threshold(dim, ma.least) + case MatchStrategy.SET_MEMBERSHIP: + return self._coalesce_set(dim, "intersection") + case MatchStrategy.SET_EXCLUSION: + return self._coalesce_set(dim, "union") case _: raise ValueError( f"Strategy {dim.match_strategy.name} not supported by accumulator" ) def compile_coalesce_na_flag(self, dim: Dimension) -> BaseExpressionAPI: - """Expression for the coalesced NA flag (1 = both sides don't-care).""" + """Expression for the coalesced NA flag (1 = combination leaves dim unconstrained).""" if dim.match_strategy == MatchStrategy.RANGE: co_min_s, co_max_s, rhs_min_s, rhs_max_s = self._range_sentinel_checks(dim) all_sentinel = ( @@ -65,6 +78,10 @@ def compile_coalesce_na_flag(self, dim: Dimension) -> BaseExpressionAPI: .__and__(rhs_max_s) ) return all_sentinel.cast(int).alias(f"co_{dim.dimension_name}_na") + if dim.match_strategy in (MatchStrategy.SET_MEMBERSHIP, MatchStrategy.SET_EXCLUSION): + co_w, rhs_w = self._set_wild_checks(dim) + field = dim.resolved_rule_field + return co_w.__and__(rhs_w).cast(int).alias(f"co_{field}_na") co_sentinel, rhs_sentinel = self._sentinel_checks(dim) field = dim.resolved_rule_field return co_sentinel.__and__(rhs_sentinel).cast(int).alias(f"co_{field}_na") @@ -177,3 +194,37 @@ def _coalesce_threshold(self, dim: Dimension, combine_fn) -> list[BaseExpression .alias(f"co_{field}") ) return [new_val] + + def _set_wild_checks(self, dim: Dimension) -> tuple[BaseExpressionAPI, BaseExpressionAPI]: + field = dim.resolved_rule_field + co_w = set_wildcard_predicate(dim, ma.col(f"co_{field}")) + rhs_w = set_wildcard_predicate(dim, ma.col(f"{field}_rhs")) + return co_w, rhs_w + + def _compatible_set_membership(self, dim: Dimension) -> BaseExpressionAPI: + co_w, rhs_w = self._set_wild_checks(dim) + field = dim.resolved_rule_field + intersection_nonempty = ( + ma.col(f"co_{field}") + .list.set_intersection(ma.col(f"{field}_rhs")) + .list.len() + .gt(ma.lit(0)) + ) + return co_w.__or__(rhs_w).__or__(intersection_nonempty) + + def _coalesce_set(self, dim: Dimension, op: str) -> list[BaseExpressionAPI]: + co_w, rhs_w = self._set_wild_checks(dim) + field = dim.resolved_rule_field + co = ma.col(f"co_{field}") + rhs = ma.col(f"{field}_rhs") + combined = ( + co.list.set_intersection(rhs) if op == "intersection" else co.list.set_union(rhs) + ) + new_val = ( + ma.when(co_w.__and__(rhs_w)).then(sentinel_list_expr(dim)) + .when(co_w).then(rhs) + .when(rhs_w).then(co) + .otherwise(canonicalize_set_expr(combined)) + .alias(f"co_{field}") + ) + return [new_val] diff --git a/tests/accumulator/test_compiler.py b/tests/accumulator/test_compiler.py index 1dc8bbd..39d3aea 100644 --- a/tests/accumulator/test_compiler.py +++ b/tests/accumulator/test_compiler.py @@ -7,6 +7,7 @@ from mountainash_rules.engines.accumulator.compiler import AccumulatorCompiler from mountainash_rules.core.constants import UNKNOWN, UNKNOWN_NUMERIC, MatchStrategy +from mountainash_rules.core.constants import DataType from mountainash_rules.core.dimension import Dimension @@ -279,3 +280,98 @@ def test_sentinel_rhs_uses_lhs(self, compiler): df = pl.DataFrame({"co_cap": [100], "cap_rhs": [UNKNOWN_NUMERIC]}) result = df.with_columns(exprs[0].compile(df, booleanizer=None)) assert result["co_cap"].to_list() == [100] + + +class TestSetMembershipCompatible: + def _dim(self): + return Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.STR) + + def test_non_empty_intersection_compatible(self, compiler): + expr = compiler.compile_compatible(self._dim()) + df = pl.DataFrame({ + "co_region": pl.Series("co_region", [["AU", "NZ"], ["AU", "NZ"]], dtype=pl.List(pl.Utf8)), + "region_rhs": pl.Series("region_rhs", [["NZ", "UK"], ["US", "CA"]], dtype=pl.List(pl.Utf8)), + }) + out = df.with_columns(expr.alias("c").compile(df, booleanizer=None)) + assert out["c"].to_list() == [True, False] + + def test_wildcard_either_side_compatible(self, compiler): + expr = compiler.compile_compatible(self._dim()) + df = pl.DataFrame({ + "co_region": pl.Series("co_region", [[""], ["AU"]], dtype=pl.List(pl.Utf8)), + "region_rhs": pl.Series("region_rhs", [["US"], [""]], dtype=pl.List(pl.Utf8)), + }) + out = df.with_columns(expr.alias("c").compile(df, booleanizer=None)) + assert out["c"].to_list() == [True, True] + + +class TestSetMembershipCoalesce: + def _dim(self): + return Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.STR) + + def test_intersection_canonicalized(self, compiler): + exprs = compiler.compile_coalesce(self._dim()) + assert len(exprs) == 1 + df = pl.DataFrame({ + "co_region": pl.Series("co_region", [["AU", "NZ", "UK"]], dtype=pl.List(pl.Utf8)), + "region_rhs": pl.Series("region_rhs", [["UK", "NZ", "US"]], dtype=pl.List(pl.Utf8)), + }) + out = df.with_columns(exprs[0].compile(df, booleanizer=None)) + assert out["co_region"].to_list() == [["NZ", "UK"]] # sorted-unique + + def test_wildcard_passthrough(self, compiler): + exprs = compiler.compile_coalesce(self._dim()) + df = pl.DataFrame({ + "co_region": pl.Series("co_region", [[""], ["AU"]], dtype=pl.List(pl.Utf8)), + "region_rhs": pl.Series("region_rhs", [["AU"], [""]], dtype=pl.List(pl.Utf8)), + }) + out = df.with_columns(exprs[0].compile(df, booleanizer=None)) + assert out["co_region"].to_list() == [["AU"], ["AU"]] + + def test_both_wildcard_stays_sentinel(self, compiler): + exprs = compiler.compile_coalesce(self._dim()) + df = pl.DataFrame({ + "co_region": pl.Series("co_region", [[""]], dtype=pl.List(pl.Utf8)), + "region_rhs": pl.Series("region_rhs", [[""]], dtype=pl.List(pl.Utf8)), + }) + out = df.with_columns(exprs[0].compile(df, booleanizer=None)) + assert out["co_region"].to_list() == [[""]] + assert out["co_region"].dtype == pl.List(pl.Utf8) + + +class TestSetExclusionCoalesce: + def _dim(self): + return Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_EXCLUSION, data_type=DataType.STR) + + def test_union_canonicalized(self, compiler): + exprs = compiler.compile_coalesce(self._dim()) + df = pl.DataFrame({ + "co_region": pl.Series("co_region", [["AU", "NZ"]], dtype=pl.List(pl.Utf8)), + "region_rhs": pl.Series("region_rhs", [["NZ", "US"]], dtype=pl.List(pl.Utf8)), + }) + out = df.with_columns(exprs[0].compile(df, booleanizer=None)) + assert out["co_region"].to_list() == [["AU", "NZ", "US"]] + + def test_always_compatible(self, compiler): + expr = compiler.compile_compatible(self._dim()) + df = pl.DataFrame({ + "co_region": pl.Series("co_region", [["AU"], [""]], dtype=pl.List(pl.Utf8)), + "region_rhs": pl.Series("region_rhs", [["NZ"], ["US"]], dtype=pl.List(pl.Utf8)), + }) + out = df.with_columns(expr.alias("c").compile(df, booleanizer=None)) + assert out["c"].to_list() == [True, True] + + +class TestSetNaFlag: + def _dim(self, strategy): + return Dimension(dimension_name="region", match_strategy=strategy, data_type=DataType.STR) + + def test_na_flag_from_final_coalesced_value(self, compiler): + # wildcard+wildcard -> 1 ; wildcard+concrete -> 0 ; concrete+concrete -> 0 + expr = compiler.compile_coalesce_na_flag(self._dim(MatchStrategy.SET_MEMBERSHIP)) + df = pl.DataFrame({ + "co_region": pl.Series("co_region", [[""], [""], ["AU"]], dtype=pl.List(pl.Utf8)), + "region_rhs": pl.Series("region_rhs", [[""], ["AU"], ["NZ"]], dtype=pl.List(pl.Utf8)), + }) + out = df.with_columns(expr.compile(df, booleanizer=None)) + assert out["co_region_na"].to_list() == [1, 0, 0] From 7bd15cd83ee1884b587300ff17c9b8cc9192e3dd Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 20 Jul 2026 14:17:29 +1000 Subject: [PATCH 09/10] feat(accumulator): set-column normalization stage; frontier untouched, dedupes correctly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Normalize every set rule column to the non-null canonical in-band-sentinel form at ingestion (before the empty-frame branch and anchor), plus a seed-NA set branch and a pre-frontier non-null safety assertion. The frontier self-join is unchanged — its inputs are now null-free and canonical, so wildcard-set combinations dedupe (3 wildcard rules collapse to prime-product {30}). The pre-frontier assertion counts nulls via all_combos.filter(...).count_rows() (all_combos is already a Relation; do not re-wrap, and count_rows is portable). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../engines/accumulator/engine.py | 66 ++++++++++++++-- tests/accumulator/test_engine.py | 78 ++++++++++++++++++- 2 files changed, 137 insertions(+), 7 deletions(-) diff --git a/src/mountainash_rules/engines/accumulator/engine.py b/src/mountainash_rules/engines/accumulator/engine.py index 6ca0325..ef82bfe 100644 --- a/src/mountainash_rules/engines/accumulator/engine.py +++ b/src/mountainash_rules/engines/accumulator/engine.py @@ -25,6 +25,12 @@ unknown_sentinel_for, ) from mountainash_rules.core.dimension import Dimension, DimensionsMetadata +from mountainash_rules.core.set_wildcard import ( + validate_set_columns, + validate_set_no_null_elements, + normalize_set_expr, + set_wildcard_predicate, +) from mountainash_rules.engines.filter.engine import ExpressionRulesEngine from mountainash_rules.engines.accumulator.lattice import Lattice from mountainash_rules.engines.accumulator.primes import ( @@ -120,6 +126,7 @@ def build( # Materialize to polars for prime injection rules_pl = rel.to_polars() + rules_pl = self._normalize_set_columns(rules_pl) n_rules = len(rules_pl) if n_rules == 0: @@ -168,6 +175,7 @@ def build( all_combos = concat(all_levels) # Step 5: Frontier filter — remove dominated combinations + self._assert_set_columns_non_null(all_combos) result = self._frontier_filter(all_combos) return Lattice( @@ -217,6 +225,45 @@ def _tracking_columns(self) -> list[str]: """All tracking column names.""" return ["__prime", "__prime_product", "__level"] + self._agg_fields() + def _set_dims(self) -> list[Dimension]: + return [ + d for d in self._constraint_dims + if d.match_strategy in (MatchStrategy.SET_MEMBERSHIP, MatchStrategy.SET_EXCLUSION) + ] + + def _normalize_set_columns(self, rules_pl: t.Any) -> t.Any: + """Validate + normalize every set-dimension rule column to the non-null, + canonical in-band-sentinel form. Runs for EVERY build path before the + empty-frame branch and the anchor, so set co_ columns are never null.""" + set_dims = self._set_dims() + if not set_dims: + return rules_pl + rel = relation(rules_pl) + validate_set_columns(rel, set_dims) # reservation (portable) + validate_set_no_null_elements(rel, set_dims) # element-nulls (polars build only) + rel = rel.with_columns(*[ + normalize_set_expr(dim, ma.col(dim.resolved_rule_field)).alias(dim.resolved_rule_field) + for dim in set_dims + ]) + return rel.to_polars() + + def _assert_set_columns_non_null(self, all_combos: t.Any) -> None: + """Safety net for the frontier 'no change' invariant: every set co_ column + must be non-null before the dominance self-join (null keys silently defeat + pruning). Runs only when set dims are present.""" + set_dims = self._set_dims() + if not set_dims: + return + # all_combos is already a mountainash Relation (concat of levels); do not + # re-wrap it. count_rows() is the portable row count (backend-pure). + cols = [f"co_{d.resolved_rule_field}" for d in set_dims] + for c in cols: + if all_combos.filter(ma.col(c).is_null()).count_rows() > 0: + raise AssertionError( + f"set co_ column {c!r} contains null before frontier filter — " + f"normalization did not reach every build path" + ) + def _create_anchor(self, rules_pl: pl.DataFrame) -> t.Any: """Create level-0 singleton combinations from rules.""" # Start with all original columns plus __prime @@ -249,12 +296,19 @@ def _create_anchor(self, rules_pl: pl.DataFrame) -> t.Any: ) else: field = dim.resolved_rule_field - sentinel = unknown_sentinel_for(dim.data_type) - na_exprs.append( - ma.col(field).eq(ma.lit(sentinel)) - .cast(int) - .alias(f"co_{field}_na") - ) + if dim.match_strategy in (MatchStrategy.SET_MEMBERSHIP, MatchStrategy.SET_EXCLUSION): + na_exprs.append( + set_wildcard_predicate(dim, ma.col(field)) + .cast(int) + .alias(f"co_{field}_na") + ) + else: + sentinel = unknown_sentinel_for(dim.data_type) + na_exprs.append( + ma.col(field).eq(ma.lit(sentinel)) + .cast(int) + .alias(f"co_{field}_na") + ) # Add tracking columns tracking_exprs = [ diff --git a/tests/accumulator/test_engine.py b/tests/accumulator/test_engine.py index e004a20..fbf05f4 100644 --- a/tests/accumulator/test_engine.py +++ b/tests/accumulator/test_engine.py @@ -6,7 +6,7 @@ from mountainash_rules.engines.accumulator.engine import AccumulatorEngine from mountainash_rules.engines.accumulator.aggregate import Aggregate -from mountainash_rules.core.constants import UNKNOWN, UNKNOWN_NUMERIC, MatchStrategy, DimensionRole +from mountainash_rules.core.constants import DataType, UNKNOWN, UNKNOWN_NUMERIC, MatchStrategy, DimensionRole from mountainash_rules.core.dimension import Dimension, DimensionsMetadata @@ -139,6 +139,82 @@ def test_all_wildcard_rules_combine_fully(self): assert rows["__agg_margin"][0] == pytest.approx(6.0) +class TestSetMembershipBuildFrontier: + def _metadata(self): + return DimensionsMetadata(dimensions=[ + Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.STR), + ]) + + def test_three_wildcard_rules_collapse_to_single_maximal(self): + # THE anchor regression: 3 wildcard-set rules must dedupe to pp=30, NOT 7 combos. + rules = pl.DataFrame({ + "rule_name": ["R1", "R2", "R3"], + "region": pl.Series("region", [None, None, None], dtype=pl.List(pl.Utf8)), + }) + engine = AccumulatorEngine(dimension_metadata=self._metadata()) + rows = _rows(engine.build(rules).combinations) + assert set(rows["__prime_product"]) == {30} + + def test_two_membership_rules_coalesce_to_intersection(self): + rules = pl.DataFrame({ + "rule_name": ["R1", "R2"], + "region": pl.Series("region", [["AU", "NZ", "UK"], ["NZ", "UK", "US"]], dtype=pl.List(pl.Utf8)), + }) + engine = AccumulatorEngine(dimension_metadata=self._metadata()) + rows = _rows(engine.build(rules).combinations) + by_pp = dict(zip(rows["__prime_product"], rows["co_region"])) + assert sorted(by_pp[6]) == ["NZ", "UK"] + + def test_same_set_different_order_dedupes(self): + # Ordering: two rules whose sets are equal up to order must dedupe to the + # single maximal combination — assert the EXACT surviving prime-product set. + rules = pl.DataFrame({ + "rule_name": ["R1", "R2"], + "region": pl.Series("region", [["UK", "NZ"], ["NZ", "UK"]], dtype=pl.List(pl.Utf8)), + }) + engine = AccumulatorEngine(dimension_metadata=self._metadata()) + rows = _rows(engine.build(rules).combinations) + # R1 and R2 have equal (canonicalized) sets, are compatible (non-empty + # intersection), so {R1,R2} (pp=6) dominates both singletons {2},{3}. + assert set(rows["__prime_product"]) == {6} + by_pp = dict(zip(rows["__prime_product"], rows["co_region"])) + assert by_pp[6] == ["NZ", "UK"] # canonical (sorted-unique) + + +class TestSetExclusionBuild: + def _metadata(self): + return DimensionsMetadata(dimensions=[ + Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_EXCLUSION, data_type=DataType.STR), + ]) + + def test_two_exclusion_rules_coalesce_to_union(self): + rules = pl.DataFrame({ + "rule_name": ["R1", "R2"], + "region": pl.Series("region", [["AU", "NZ"], ["NZ", "US"]], dtype=pl.List(pl.Utf8)), + }) + engine = AccumulatorEngine(dimension_metadata=self._metadata()) + rows = _rows(engine.build(rules).combinations) + by_pp = dict(zip(rows["__prime_product"], rows["co_region"])) + assert by_pp[6] == ["AU", "NZ", "US"] + + +class TestSetBuildValidation: + def _metadata(self): + return DimensionsMetadata(dimensions=[ + Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.STR), + ]) + + def test_embedded_sentinel_rejected(self): + import pytest + rules = pl.DataFrame({ + "rule_name": ["R1"], + "region": pl.Series("region", [["AU", ""]], dtype=pl.List(pl.Utf8)), + }) + engine = AccumulatorEngine(dimension_metadata=self._metadata()) + with pytest.raises(ValueError, match="sentinel"): + engine.build(rules) + + class TestBuildWithPartitionKey: def test_partition_filters_rules(self): rules = pl.DataFrame({ From 632c079c7090de7eea121c4d510e9eda9636f5e1 Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Mon, 20 Jul 2026 14:22:32 +1000 Subject: [PATCH 10/10] test+docs(accumulator): set-wildcard apply round-trip, float typing; document in-band sentinel Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 4 +++- tests/accumulator/test_apply.py | 29 ++++++++++++++++++++++++++++- tests/accumulator/test_engine.py | 23 +++++++++++++++++++++++ 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ab82e65..70d57e8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,6 +24,8 @@ Wildcards are in-band typed sentinels (see `constants.sentinels_for(data_type)`) | date | `date(1,1,1)` | `date(1,1,2)` | | datetime | `datetime(1,1,1)` | `datetime(1,1,2)` | +Set-dimension wildcards use the in-band `[sentinel]` list (not null), normalized at ingestion (reservation check in both engines; element-null check in the accumulator build). + ### Filter engine pipeline (`engine.py`) 1. **Compile** — `DimensionCompiler` (`compiler.py`) turns each `Dimension` into a backend-agnostic ternary expression template at construction. @@ -67,7 +69,7 @@ No module under `src/mountainash_rules/` may import polars/ibis/narwhals directl | `prefix` / `suffix` / `contains` | string | | | `regex` | per-row pattern column | Polars-native fallback (`# allow:` tagged) | | `context_regex` | literal `regex_pattern` on the Dimension | global context validator | -| `set_membership` / `set_exclusion` | list column | Polars-native fallback | +| `set_membership` / `set_exclusion` | list column | Filter via `t_is_in`/`t_is_not_in` (polars/ibis; narwhals list ops under `mountainash#89`). Accumulator-coalesceable (membership → list intersection, exclusion → list union). Wildcard = in-band `[unknown_sentinel_for(dtype)]` (never null; bool unsupported); see `null-is-not-a-portable-sentinel` principle. | **Adding a strategy:** add enum value in `core/constants.py`, validation in `core/dimension.py`, `_compile_` in `core/compiler.py`, test class in `tests/core/test_compiler.py`. diff --git a/tests/accumulator/test_apply.py b/tests/accumulator/test_apply.py index 6ff3f37..d0839a8 100644 --- a/tests/accumulator/test_apply.py +++ b/tests/accumulator/test_apply.py @@ -8,7 +8,7 @@ from mountainash_rules.engines.accumulator.engine import AccumulatorEngine from mountainash_rules.engines.accumulator.result import AccumulatorResult from mountainash_rules.engines.accumulator.aggregate import Aggregate -from mountainash_rules.core.constants import UNKNOWN, UNKNOWN_NUMERIC, NOT_SET_NUMERIC, MatchStrategy, DimensionRole +from mountainash_rules.core.constants import DataType, UNKNOWN, UNKNOWN_NUMERIC, NOT_SET_NUMERIC, MatchStrategy, DimensionRole from mountainash_rules.core.dimension import Dimension, DimensionsMetadata @@ -246,3 +246,30 @@ def test_apply_auto_skips_load_validation(self): lattices, {"region": "AU", "channel": "DIRECT", "product": "GOLD"} ) assert result.count >= 1 + + +class TestSetMembershipApply: + def _metadata(self): + return DimensionsMetadata(dimensions=[ + Dimension(dimension_name="region", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.STR), + ]) + + def test_context_in_intersection_matches_combination(self): + rules = pl.DataFrame({ + "rule_name": ["R1", "R2"], + "region": pl.Series("region", [["AU", "NZ", "UK"], ["NZ", "UK", "US"]], dtype=pl.List(pl.Utf8)), + }) + engine = AccumulatorEngine(dimension_metadata=self._metadata()) + lattice = engine.build(rules) + result = engine.apply(lattice, {"region": "NZ"}) + assert 6 in set(_rows(result.provenance)["__prime_product"]) + + def test_wildcard_combination_matches_any_context_exact_count(self): + rules = pl.DataFrame({ + "rule_name": ["R1"], + "region": pl.Series("region", [None], dtype=pl.List(pl.Utf8)), + }) + engine = AccumulatorEngine(dimension_metadata=self._metadata()) + lattice = engine.build(rules) + result = engine.apply(lattice, {"region": "ANYTHING"}) + assert result.count == 1 # exactly the single wildcard combination diff --git a/tests/accumulator/test_engine.py b/tests/accumulator/test_engine.py index fbf05f4..aed2ad4 100644 --- a/tests/accumulator/test_engine.py +++ b/tests/accumulator/test_engine.py @@ -215,6 +215,29 @@ def test_embedded_sentinel_rejected(self): engine.build(rules) +class TestFloatSetDimensionBuild: + def _metadata(self): + return DimensionsMetadata(dimensions=[ + Dimension(dimension_name="scores", match_strategy=MatchStrategy.SET_MEMBERSHIP, data_type=DataType.FLOAT), + ]) + + def test_float_set_wildcard_and_coalesce(self): + rules = pl.DataFrame({ + "rule_name": ["R1", "R2"], + "scores": pl.Series("scores", [[1.5, 2.5, 3.5], None], dtype=pl.List(pl.Float64)), + }) + engine = AccumulatorEngine(dimension_metadata=self._metadata()) + lattice = engine.build(rules) + rows = _rows(lattice.combinations) + by_pp = dict(zip(rows["__prime_product"], rows["co_scores"])) + # R2 is a wildcard; {R1,R2} coalesces to R1's concrete set (wildcard passthrough). + assert by_pp[6] == [1.5, 2.5, 3.5] + # Column stays a Float list — verify no dtype collapse. + import polars as _pl + mat = relation(lattice.combinations).to_polars() + assert mat.schema["co_scores"] == _pl.List(_pl.Float64) + + class TestBuildWithPartitionKey: def test_partition_filters_rules(self): rules = pl.DataFrame({