feat: set-dimension wildcard as in-band sentinel (frontier-dedup fix)

CLAUDE.md

@@ -24,6 +24,8 @@ Wildcards are in-band typed sentinels (see `constants.sentinels_for(data_type)`)
 | date | `date(1,1,1)` | `date(1,1,2)` |
 | datetime | `datetime(1,1,1)` | `datetime(1,1,2)` |
 
+Set-dimension wildcards use the in-band `[sentinel]` list (not null), normalized at ingestion (reservation check in both engines; element-null check in the accumulator build).
+
 ### Filter engine pipeline (`engine.py`)
 
 1. **Compile** — `DimensionCompiler` (`compiler.py`) turns each `Dimension` into a backend-agnostic ternary expression template at construction.
@@ -67,7 +69,7 @@ No module under `src/mountainash_rules/` may import polars/ibis/narwhals directl
 | `prefix` / `suffix` / `contains` | string | |
 | `regex` | per-row pattern column | Polars-native fallback (`# allow:` tagged) |
 | `context_regex` | literal `regex_pattern` on the Dimension | global context validator |
-| `set_membership` / `set_exclusion` | list column | Polars-native fallback |
+| `set_membership` / `set_exclusion` | list column | Filter via `t_is_in`/`t_is_not_in` (polars/ibis; narwhals list ops under `mountainash#89`). Accumulator-coalesceable (membership → list intersection, exclusion → list union). Wildcard = in-band `[unknown_sentinel_for(dtype)]` (never null; bool unsupported); see `null-is-not-a-portable-sentinel` principle. |
 
 **Adding a strategy:** add enum value in `core/constants.py`, validation in `core/dimension.py`, `_compile_<strategy>` in `core/compiler.py`, test class in `tests/core/test_compiler.py`.
 

src/mountainash_rules/core/compiler.py

@@ -15,6 +15,7 @@
     unknown_sentinel_for,
 )
 from mountainash_rules.core.dimension import Dimension, DimensionsMetadata
+from mountainash_rules.core.set_wildcard import normalize_set_expr, set_wildcard_predicate
 
 
 class DimensionCompiler:
@@ -208,15 +209,15 @@ def _compile_context_regex(self, dim: Dimension) -> BaseExpressionAPI:
         return ma.when(match).then(1).otherwise(-1)
 
     def _compile_set_membership(self, dim: Dimension) -> BaseExpressionAPI:
-        """SET_MEMBERSHIP: context value is in the rule's list column."""
-        sentinels = sentinels_for(dim.data_type)
-        rule_col = ma.col(dim.resolved_rule_field)
-        ctx_col = ma.t_col(CTX_PREFIX + dim.dimension_name, unknown=sentinels)
-        return ctx_col.t_is_in(rule_col)
+        """SET_MEMBERSHIP: context value in the rule list; wildcard rule -> ternary 0."""
+        rule_col = normalize_set_expr(dim, ma.col(dim.resolved_rule_field))
+        ctx_col = ma.t_col(CTX_PREFIX + dim.dimension_name, unknown=sentinels_for(dim.data_type))
+        is_wild = set_wildcard_predicate(dim, rule_col)
+        return ma.when(is_wild).then(0).otherwise(ctx_col.t_is_in(rule_col))
 
     def _compile_set_exclusion(self, dim: Dimension) -> BaseExpressionAPI:
-        """SET_EXCLUSION: context value is NOT in the rule's list column."""
-        sentinels = sentinels_for(dim.data_type)
-        rule_col = ma.col(dim.resolved_rule_field)
-        ctx_col = ma.t_col(CTX_PREFIX + dim.dimension_name, unknown=sentinels)
-        return ctx_col.t_is_not_in(rule_col)
+        """SET_EXCLUSION: context value NOT in the rule list; wildcard rule -> ternary 0."""
+        rule_col = normalize_set_expr(dim, ma.col(dim.resolved_rule_field))
+        ctx_col = ma.t_col(CTX_PREFIX + dim.dimension_name, unknown=sentinels_for(dim.data_type))
+        is_wild = set_wildcard_predicate(dim, rule_col)
+        return ma.when(is_wild).then(0).otherwise(ctx_col.t_is_not_in(rule_col))

src/mountainash_rules/core/dimension.py

@@ -72,7 +72,7 @@
         return self.rule_field or self.dimension_name
 
     @model_validator(mode="after")
     def _validate_strategy_fields(self) -> "Dimension":
         orderable = self.data_type.is_numeric or self.data_type.is_temporal
 
         if self.match_strategy == MatchStrategy.RANGE:
@@ -131,6 +131,18 @@
                     f"temporal type"
                 )
 
+        if self.match_strategy in (
+            MatchStrategy.SET_MEMBERSHIP,
+            MatchStrategy.SET_EXCLUSION,
+        ):
+            if self.data_type is DataType.BOOL:
+                raise ValueError(
+                    f"Dimension '{self.dimension_name}' uses "
+                    f"{self.match_strategy.value} with data_type bool; boolean "
+                    f"set dimensions are not supported (no typed wildcard sentinel "
+                    f"exists and a set over {{true, false}} is degenerate)"
+                )
+
         return self
 
 

src/mountainash_rules/core/set_wildcard.py

@@ -0,0 +1,122 @@
+"""Shared in-band sentinel representation for set-dimension wildcards.
+
+A SET_MEMBERSHIP/SET_EXCLUSION wildcard is the single-element list
+``[unknown_sentinel_for(dim.data_type)]`` — never a null. These helpers are the
+single source of truth for both the filter engine (``core/compiler.py``) and the
+accumulator engine (``engines/accumulator/``), so the two cannot diverge.
+
+The sentinel is a reserved, out-of-domain value; ``validate_set_columns``
+enforces that a concrete rule list never embeds it, so ``set_wildcard_predicate``
+is unambiguous.
+"""
+
+from __future__ import annotations
+
+import typing as t
+
+import mountainash.expressions as ma
+from mountainash.expressions import BaseExpressionAPI
+
+from mountainash_rules.core.constants import DataType, unknown_sentinel_for
+from mountainash_rules.core.dimension import Dimension
+
+
+def _typed_sentinel(dim: Dimension) -> t.Any:
+    """The dimension's wildcard sentinel, coerced to the dim's Python element type."""
+    sent = unknown_sentinel_for(dim.data_type)
+    return float(sent) if dim.data_type == DataType.FLOAT else sent
+
+
+def sentinel_list_expr(dim: Dimension) -> BaseExpressionAPI:
+    """A ``[sentinel]`` list literal whose element carries the dim's Python type.
+
+    Element coercion (not a native list-dtype cast) keeps this backend-pure: a
+    Python-float element yields a ``List(Float64)`` literal with no cast.
+    """
+    return ma.lit([_typed_sentinel(dim)])
+
+
+def canonicalize_set_expr(col: BaseExpressionAPI) -> BaseExpressionAPI:
+    """Sort + dedupe a list column so equal sets compare/fingerprint identically."""
+    return col.list.unique().list.sort()
+
+
+def normalize_set_expr(dim: Dimension, col: BaseExpressionAPI) -> BaseExpressionAPI:
+    """Null list -> ``[sentinel]``; concrete list -> sorted-unique. The one normaliser.
+
+    Idempotent: re-applying to an already-normalized column is a no-op.
+    """
+    return ma.when(col.is_null()).then(sentinel_list_expr(dim)).otherwise(
+        canonicalize_set_expr(col)
+    )
+
+
+def set_wildcard_predicate(dim: Dimension, col: BaseExpressionAPI) -> BaseExpressionAPI:
+    """True when ``col`` (post-normalization) is the wildcard.
+
+    Because ``validate_set_columns`` rejects any concrete list embedding the
+    sentinel, ``list.contains(sentinel)`` is true iff the list is exactly
+    ``[sentinel]``. Must be evaluated after normalization (``contains`` returns
+    null on a null cell).
+    """
+    return col.list.contains(ma.lit(_typed_sentinel(dim)))
+
+
+def _embedded_sentinel_predicate(dim: Dimension, field: str) -> BaseExpressionAPI:
+    """True for a non-null list that contains the sentinel but is not ``[sentinel]``."""
+    col = ma.col(field)
+    has_sentinel = col.list.contains(ma.lit(_typed_sentinel(dim)))
+    length = col.list.len()
+    return col.is_not_null().__and__(has_sentinel.__and__(length.ne(ma.lit(1))))
+
+
+def _null_element_predicate(field: str) -> BaseExpressionAPI:
+    """True for a non-null list that contains a null element.
+
+    Uses ``list.drop_nulls`` — mountainash's IBIS backend raises
+    ``BackendCapabilityError`` for this op, so callers must only run this on the
+    polars-internal accumulator build path (see ``validate_set_no_null_elements``).
+    """
+    col = ma.col(field)
+    length = col.list.len()
+    non_null_length = col.list.drop_nulls().list.len()
+    return col.is_not_null().__and__(length.ne(non_null_length))
+
+
+def validate_set_columns(rules_rel: t.Any, set_dims: list[Dimension]) -> None:
+    """Raise ``ValueError`` if a concrete set-rule list embeds the reserved sentinel.
+
+    Portable — uses only ``list.contains`` + ``list.len`` (Ibis/Narwhals-safe), so
+    it runs in BOTH engines on any backend. A whole-list null is valid (the
+    wildcard). ``rules_rel`` is a ``mountainash`` relation; row existence is tested
+    with the relation's portable ``count_rows`` (backend-pure — no native import;
+    builtin ``len`` on a collected frame is NOT portable — Ibis rejects it).
+    """
+    for dim in set_dims:
+        field = dim.resolved_rule_field
+        n_embedded = rules_rel.filter(_embedded_sentinel_predicate(dim, field)).count_rows()
+        if n_embedded > 0:
+            raise ValueError(
+                f"Dimension '{dim.dimension_name}': a concrete rule list embeds the "
+                f"reserved wildcard sentinel {unknown_sentinel_for(dim.data_type)!r}. "
+                f"The sentinel is only valid as the sole element (the wildcard)."
+            )
+
+
+def validate_set_no_null_elements(rules_rel: t.Any, set_dims: list[Dimension]) -> None:
+    """Raise ``ValueError`` if a set-rule list contains a null element.
+
+    Uses ``list.drop_nulls`` (Ibis-unsupported), so this is called ONLY on the
+    polars-internal accumulator build path. The filter engine does not call it:
+    ``t_is_in`` tolerates a null element (it matches nothing), so a standalone
+    filter engine over Ibis set rules is unaffected.
+    """
+    for dim in set_dims:
+        field = dim.resolved_rule_field
+        n_null_elem = rules_rel.filter(_null_element_predicate(field)).count_rows()
+        if n_null_elem > 0:
+            raise ValueError(
+                f"Dimension '{dim.dimension_name}': a rule list contains a null "
+                f"element. Element-level nulls are not allowed; use a whole-list "
+                f"null (or omit the cell) for a wildcard."
+            )

src/mountainash_rules/engines/accumulator/compiler.py

@@ -10,6 +10,11 @@
     unknown_sentinel_for,
 )
 from mountainash_rules.core.dimension import Dimension
+from mountainash_rules.core.set_wildcard import (
+    set_wildcard_predicate,
+    canonicalize_set_expr,
+    sentinel_list_expr,
+)
 
 
 class AccumulatorCompiler:
@@ -34,6 +39,10 @@ def compile_compatible(self, dim: Dimension) -> BaseExpressionAPI:
                 return self._compatible_range(dim)
             case MatchStrategy.GREATER_THAN | MatchStrategy.LESS_THAN:
                 return self._compatible_threshold(dim)
+            case MatchStrategy.SET_MEMBERSHIP:
+                return self._compatible_set_membership(dim)
+            case MatchStrategy.SET_EXCLUSION:
+                return ma.lit(True)
             case _:
                 raise ValueError(
                     f"Strategy {dim.match_strategy.name} not supported by accumulator"
@@ -50,13 +59,17 @@ def compile_coalesce(self, dim: Dimension) -> list[BaseExpressionAPI]:
                 return self._coalesce_threshold(dim, ma.greatest)
             case MatchStrategy.LESS_THAN:
                 return self._coalesce_threshold(dim, ma.least)
+            case MatchStrategy.SET_MEMBERSHIP:
+                return self._coalesce_set(dim, "intersection")
+            case MatchStrategy.SET_EXCLUSION:
+                return self._coalesce_set(dim, "union")
             case _:
                 raise ValueError(
                     f"Strategy {dim.match_strategy.name} not supported by accumulator"
                 )
 
     def compile_coalesce_na_flag(self, dim: Dimension) -> BaseExpressionAPI:
-        """Expression for the coalesced NA flag (1 = both sides don't-care)."""
+        """Expression for the coalesced NA flag (1 = combination leaves dim unconstrained)."""
         if dim.match_strategy == MatchStrategy.RANGE:
             co_min_s, co_max_s, rhs_min_s, rhs_max_s = self._range_sentinel_checks(dim)
             all_sentinel = (
@@ -65,6 +78,10 @@ def compile_coalesce_na_flag(self, dim: Dimension) -> BaseExpressionAPI:
                 .__and__(rhs_max_s)
             )
             return all_sentinel.cast(int).alias(f"co_{dim.dimension_name}_na")
+        if dim.match_strategy in (MatchStrategy.SET_MEMBERSHIP, MatchStrategy.SET_EXCLUSION):
+            co_w, rhs_w = self._set_wild_checks(dim)
+            field = dim.resolved_rule_field
+            return co_w.__and__(rhs_w).cast(int).alias(f"co_{field}_na")
         co_sentinel, rhs_sentinel = self._sentinel_checks(dim)
         field = dim.resolved_rule_field
         return co_sentinel.__and__(rhs_sentinel).cast(int).alias(f"co_{field}_na")
@@ -177,3 +194,37 @@ def _coalesce_threshold(self, dim: Dimension, combine_fn) -> list[BaseExpression
             .alias(f"co_{field}")
         )
         return [new_val]
+
+    def _set_wild_checks(self, dim: Dimension) -> tuple[BaseExpressionAPI, BaseExpressionAPI]:
+        field = dim.resolved_rule_field
+        co_w = set_wildcard_predicate(dim, ma.col(f"co_{field}"))
+        rhs_w = set_wildcard_predicate(dim, ma.col(f"{field}_rhs"))
+        return co_w, rhs_w
+
+    def _compatible_set_membership(self, dim: Dimension) -> BaseExpressionAPI:
+        co_w, rhs_w = self._set_wild_checks(dim)
+        field = dim.resolved_rule_field
+        intersection_nonempty = (
+            ma.col(f"co_{field}")
+            .list.set_intersection(ma.col(f"{field}_rhs"))
+            .list.len()
+            .gt(ma.lit(0))
+        )
+        return co_w.__or__(rhs_w).__or__(intersection_nonempty)
+
+    def _coalesce_set(self, dim: Dimension, op: str) -> list[BaseExpressionAPI]:
+        co_w, rhs_w = self._set_wild_checks(dim)
+        field = dim.resolved_rule_field
+        co = ma.col(f"co_{field}")
+        rhs = ma.col(f"{field}_rhs")
+        combined = (
+            co.list.set_intersection(rhs) if op == "intersection" else co.list.set_union(rhs)
+        )
+        new_val = (
+            ma.when(co_w.__and__(rhs_w)).then(sentinel_list_expr(dim))
+            .when(co_w).then(rhs)
+            .when(rhs_w).then(co)
+            .otherwise(canonicalize_set_expr(combined))
+            .alias(f"co_{field}")
+        )
+        return [new_val]

src/mountainash_rules/engines/accumulator/engine.py

@@ -25,6 +25,12 @@
     unknown_sentinel_for,
 )
 from mountainash_rules.core.dimension import Dimension, DimensionsMetadata
+from mountainash_rules.core.set_wildcard import (
+    validate_set_columns,
+    validate_set_no_null_elements,
+    normalize_set_expr,
+    set_wildcard_predicate,
+)
 from mountainash_rules.engines.filter.engine import ExpressionRulesEngine
 from mountainash_rules.engines.accumulator.lattice import Lattice
 from mountainash_rules.engines.accumulator.primes import (
@@ -120,6 +126,7 @@ def build(
 
         # Materialize to polars for prime injection
         rules_pl = rel.to_polars()
+        rules_pl = self._normalize_set_columns(rules_pl)
         n_rules = len(rules_pl)
 
         if n_rules == 0:
@@ -168,6 +175,7 @@ def build(
         all_combos = concat(all_levels)
 
         # Step 5: Frontier filter — remove dominated combinations
+        self._assert_set_columns_non_null(all_combos)
         result = self._frontier_filter(all_combos)
 
         return Lattice(
@@ -217,6 +225,45 @@ def _tracking_columns(self) -> list[str]:
         """All tracking column names."""
         return ["__prime", "__prime_product", "__level"] + self._agg_fields()
 
+    def _set_dims(self) -> list[Dimension]:
+        return [
+            d for d in self._constraint_dims
+            if d.match_strategy in (MatchStrategy.SET_MEMBERSHIP, MatchStrategy.SET_EXCLUSION)
+        ]
+
+    def _normalize_set_columns(self, rules_pl: t.Any) -> t.Any:
+        """Validate + normalize every set-dimension rule column to the non-null,
+        canonical in-band-sentinel form. Runs for EVERY build path before the
+        empty-frame branch and the anchor, so set co_ columns are never null."""
+        set_dims = self._set_dims()
+        if not set_dims:
+            return rules_pl
+        rel = relation(rules_pl)
+        validate_set_columns(rel, set_dims)          # reservation (portable)
+        validate_set_no_null_elements(rel, set_dims)  # element-nulls (polars build only)
+        rel = rel.with_columns(*[
+            normalize_set_expr(dim, ma.col(dim.resolved_rule_field)).alias(dim.resolved_rule_field)
+            for dim in set_dims
+        ])
+        return rel.to_polars()
+
+    def _assert_set_columns_non_null(self, all_combos: t.Any) -> None:
+        """Safety net for the frontier 'no change' invariant: every set co_ column
+        must be non-null before the dominance self-join (null keys silently defeat
+        pruning). Runs only when set dims are present."""
+        set_dims = self._set_dims()
+        if not set_dims:
+            return
+        # all_combos is already a mountainash Relation (concat of levels); do not
+        # re-wrap it. count_rows() is the portable row count (backend-pure).
+        cols = [f"co_{d.resolved_rule_field}" for d in set_dims]
+        for c in cols:
+            if all_combos.filter(ma.col(c).is_null()).count_rows() > 0:
+                raise AssertionError(
+                    f"set co_ column {c!r} contains null before frontier filter — "
+                    f"normalization did not reach every build path"
+                )
+
     def _create_anchor(self, rules_pl: pl.DataFrame) -> t.Any:
         """Create level-0 singleton combinations from rules."""
         # Start with all original columns plus __prime
@@ -249,12 +296,19 @@ def _create_anchor(self, rules_pl: pl.DataFrame) -> t.Any:
                 )
             else:
                 field = dim.resolved_rule_field
-                sentinel = unknown_sentinel_for(dim.data_type)
-                na_exprs.append(
-                    ma.col(field).eq(ma.lit(sentinel))
-                    .cast(int)
-                    .alias(f"co_{field}_na")
-                )
+                if dim.match_strategy in (MatchStrategy.SET_MEMBERSHIP, MatchStrategy.SET_EXCLUSION):
+                    na_exprs.append(
+                        set_wildcard_predicate(dim, ma.col(field))
+                        .cast(int)
+                        .alias(f"co_{field}_na")
+                    )
+                else:
+                    sentinel = unknown_sentinel_for(dim.data_type)
+                    na_exprs.append(
+                        ma.col(field).eq(ma.lit(sentinel))
+                        .cast(int)
+                        .alias(f"co_{field}_na")
+                    )
 
         # Add tracking columns
         tracking_exprs = [