diff --git a/src/mountainash/core/capabilities/registry.py b/src/mountainash/core/capabilities/registry.py index 833c4c3b..490f56a8 100644 --- a/src/mountainash/core/capabilities/registry.py +++ b/src/mountainash/core/capabilities/registry.py @@ -200,6 +200,21 @@ def _validate_fact(family: CONST_BACKEND, fact: CapabilityFact) -> None: "gate_params — the fact could never gate. Add the param to the " "op's gate_params (RelationOperationDef) or declare a non-GATE enforcement role." ) + # Residue reachability (item 98): a handler-routed op's MATERIALIZE_RESIDUE + # fact can only ever fire if the handler wraps its native call via + # _enrich_native_call (relation_visitor.py) — reject a silently-dead + # declaration on an unwrapped handler at registration. + if ( + kind == "relation" + and fact.enforcement is Enforcement.MATERIALIZE_RESIDUE + and getattr(definition, "handler", None) is not None + and not getattr(definition, "wraps_native_call", False) + ): + raise ValueError( + f"CapabilityFact({fact.operation_key!r}): the op is handler-routed " + "and does not declare wraps_native_call=True, so a " + "MATERIALIZE_RESIDUE fact could never fire through per-op enrichment." + ) class CapabilityRegistry: diff --git a/src/mountainash/core/limitations.py b/src/mountainash/core/limitations.py index 69e0eb95..ca9483ee 100644 --- a/src/mountainash/core/limitations.py +++ b/src/mountainash/core/limitations.py @@ -71,6 +71,7 @@ def enrich_materialization( fn: Callable[[], Any], *, prefer_operation_keys: "frozenset | None" = None, + dialect: "str | None" = None, ) -> Any: """Materialization-boundary enrichment: consult the spine's MATERIALIZE residue (matched by native exception type — residue facts keep their @@ -89,13 +90,18 @@ def enrich_materialization( raised error is enriched only when **exactly one** candidate matches the exception's type; zero or multiple matches leave the original exception to propagate raw rather than guessing. + dialect: override the backend's own dialect for the residue lookup + — the authoritative input dialect item 95 resolves for + multi-input nodes. """ from mountainash.core.capabilities import CapabilityRegistry from mountainash.core.types import BackendCapabilityError family = getattr(backend, "backend_type", None) residue = ( - CapabilityRegistry.residue_for(family, getattr(backend, "dialect", None)) + CapabilityRegistry.residue_for( + family, dialect if dialect is not None else getattr(backend, "dialect", None) + ) if family is not None else {} ) diff --git a/src/mountainash/relations/core/relation_system/relation_mapping/definitions.py b/src/mountainash/relations/core/relation_system/relation_mapping/definitions.py index f84bc59d..541c092b 100644 --- a/src/mountainash/relations/core/relation_system/relation_mapping/definitions.py +++ b/src/mountainash/relations/core/relation_system/relation_mapping/definitions.py @@ -135,6 +135,7 @@ def _ext(key, method): substrait_rel="JoinRel", protocol_method=SubstraitJoinRelationSystemProtocol.join, handler=handlers.visit_join, # cross-backend right-side coercion + wraps_native_call=True, ), RelationOperationDef( operation_key=RS.AGGREGATE, @@ -188,6 +189,7 @@ def _ext(key, method): is_extension=True, extension_uri=MountainashRelExtension.UTIL, handler=handlers.visit_source, + wraps_native_call=True, ), RelationOperationDef( operation_key=RM.REF, @@ -203,6 +205,7 @@ def _ext(key, method): extension_uri=MountainashRelExtension.DAG, protocol_method=ExtProto.read_resource, handler=handlers.visit_resource_read, + wraps_native_call=True, ), RelationOperationDef( operation_key=RM.CONFORM, @@ -210,6 +213,7 @@ def _ext(key, method): is_extension=True, extension_uri=MountainashRelExtension.CONFORM, handler=handlers.visit_conform, + wraps_native_call=True, ), RelationOperationDef( operation_key=RM.FETCH_FROM_END, @@ -229,6 +233,7 @@ def _ext(key, method): protocol_method=ExtProto.join_asof, handler=handlers.visit_join_asof, gate_params=("tolerance",), + wraps_native_call=True, ), RelationOperationDef( # No node type: invoked from the conform path, not node dispatch. diff --git a/src/mountainash/relations/core/relation_system/relation_mapping/handlers.py b/src/mountainash/relations/core/relation_system/relation_mapping/handlers.py index 2157cfc2..9f7d435b 100644 --- a/src/mountainash/relations/core/relation_system/relation_mapping/handlers.py +++ b/src/mountainash/relations/core/relation_system/relation_mapping/handlers.py @@ -10,29 +10,37 @@ from typing import Any +from mountainash.relations.core.relation_system.relation_keys.enums import ( + RKEY_MOUNTAINASH_REL, + RKEY_SUBSTRAIT_REL, +) + def visit_join(node: Any, visitor: Any) -> Any: left = visitor.visit(node.left) right = visitor._visit_and_coerce_right(node.right, left) - return visitor.backend.join( - left, right, - join_type=node.join_type, - on=node.on, - left_on=node.left_on, - right_on=node.right_on, - suffix=node.suffix, + return visitor._enrich_native_call( + node, RKEY_SUBSTRAIT_REL.JOIN, + lambda: visitor.backend.join( + left, right, + join_type=node.join_type, + on=node.on, left_on=node.left_on, + right_on=node.right_on, suffix=node.suffix, + ), ) def visit_join_asof(node: Any, visitor: Any) -> Any: left = visitor.visit(node.left) right = visitor._visit_and_coerce_right(node.right, left) - return visitor.backend.join_asof( - left, right, - on=node.on[0] if node.on else node.left_on[0], - by=node.by, - strategy=node.strategy or "backward", - tolerance=node.tolerance, + return visitor._enrich_native_call( + node, RKEY_MOUNTAINASH_REL.JOIN_ASOF, + lambda: visitor.backend.join_asof( + left, right, + on=node.on[0] if node.on else node.left_on[0], + by=node.by, strategy=node.strategy or "backward", + tolerance=node.tolerance, + ), ) @@ -47,21 +55,29 @@ def visit_ref(node: Any, visitor: Any) -> Any: def visit_resource_read(node: Any, visitor: Any) -> Any: - out = visitor.backend.read_resource(node.resource) - if node.resource.table_schema is not None: - out = visitor.apply_conform( - out, node.resource.table_schema, empty_from_schema=True, - resource_name=node.resource.name, - ) - return out + def _read_and_conform(): + out = visitor.backend.read_resource(node.resource) + if node.resource.table_schema is not None: + out = visitor.apply_conform( + out, node.resource.table_schema, empty_from_schema=True, + resource_name=node.resource.name, + ) + return out + return visitor._enrich_native_call(node, RKEY_MOUNTAINASH_REL.READ_RESOURCE, _read_and_conform) def visit_source(node: Any, visitor: Any) -> Any: from mountainash.pydata.ingress.pydata_ingress import PydataIngress df = PydataIngress.convert(node.data) - return visitor.backend.read(df) + return visitor._enrich_native_call( + node, RKEY_MOUNTAINASH_REL.SOURCE, + lambda: visitor.backend.read(df), + ) def visit_conform(node: Any, visitor: Any) -> Any: native = visitor.visit(node.input) - return visitor.apply_conform(native, node.spec, contract=node.contract) + return visitor._enrich_native_call( + node, RKEY_MOUNTAINASH_REL.CONFORM, + lambda: visitor.apply_conform(native, node.spec, contract=node.contract), + ) diff --git a/src/mountainash/relations/core/relation_system/relation_mapping/registry.py b/src/mountainash/relations/core/relation_system/relation_mapping/registry.py index ddf5d67f..25b4b6b5 100644 --- a/src/mountainash/relations/core/relation_system/relation_mapping/registry.py +++ b/src/mountainash/relations/core/relation_system/relation_mapping/registry.py @@ -50,6 +50,7 @@ class RelationOperationDef: options_field: Optional[str] = None # dict field spread as **kwargs (ExtensionRelNode.options) handler: Optional[Callable] = None # custom compile override: (node, visitor) -> Any gate_params: tuple[str, ...] = () # extra node fields that gate capability facts + wraps_native_call: bool = False # handler op wraps its native call in _enrich_native_call def get_signature(self) -> Optional[inspect.Signature]: if self.protocol_method is None: diff --git a/src/mountainash/relations/core/unified_visitor/relation_visitor.py b/src/mountainash/relations/core/unified_visitor/relation_visitor.py index d62aa1cb..4fcd8192 100644 --- a/src/mountainash/relations/core/unified_visitor/relation_visitor.py +++ b/src/mountainash/relations/core/unified_visitor/relation_visitor.py @@ -73,15 +73,15 @@ def _iter_function_keys(value: Any, _seen: "set[int] | None" = None): yield from _iter_function_keys(item, _seen) -def _present_expression_function_keys(node: RelationNode, op: Any) -> frozenset: - """Function keys structurally present in *node*'s bound - ``EXPRESSION``/``EXPRESSION_LIST`` args -- the caller's evidence for - which operation(s) were actually being compiled, used to disambiguate - :func:`~mountainash.core.limitations.enrich_materialization` candidates - that share a native exception type.""" +def _present_operation_keys(node: RelationNode, op: Any) -> frozenset: + """The op's own RKEY plus the function keys structurally present in + *node*'s bound ``EXPRESSION``/``EXPRESSION_LIST`` args -- the caller's + evidence for which operation(s) were actually being compiled, used to + disambiguate :func:`~mountainash.core.limitations.enrich_materialization` + candidates that share a native exception type.""" from mountainash.relations.core.relation_system.relation_mapping.registry import ArgKind - keys: set = set() + keys: set = {op.operation_key} for binding in op.args: if binding.kind in (ArgKind.EXPRESSION, ArgKind.EXPRESSION_LIST): keys.update(_iter_function_keys(getattr(node, binding.field))) @@ -271,22 +271,36 @@ def _dispatch(self, node: RelationNode, op: Any) -> Any: from mountainash.core.limitations import enrich_materialization if op.handler is not None: - # No generic expression-field introspection for handler-routed - # ops today -- an empty (non-None) preferred set is - # authoritative in enrich_materialization, so this is a - # verified no-op unless/until a handler op registers a - # MATERIALIZE_RESIDUE fact. - return enrich_materialization( - self.backend, lambda: op.handler(node, self), - prefer_operation_keys=frozenset(), - ) + # Handler ops wrap their own native call (see handlers.py) -- + # never wrap the whole handler, or a child read/coercion + # TypeError would be narrowed under the parent's RKEY. + return op.handler(node, self) method = getattr(self.backend, op.protocol_method.__name__) args = [self._bind(node, b) for b in op.args] # children compiled OUTSIDE the wrap kwargs = self._bind_options(node, op) - prefer = _present_expression_function_keys(node, op) + prefer = _present_operation_keys(node, op) + d = self._authoritative_dialect(node, op) + dialect = getattr(self.backend, "dialect", None) if d is _UNRESOLVED else d return enrich_materialization( self.backend, lambda: method(*args, **kwargs), prefer_operation_keys=prefer, + dialect=dialect, + ) + + def _enrich_native_call(self, node: RelationNode, operation_key: Any, fn: Callable[[], Any]) -> Any: + """Wrap a single native backend call in residue enrichment, scoped to + ``operation_key`` and the authoritative input dialect (item 95).""" + from mountainash.core.limitations import enrich_materialization + from mountainash.relations.core.relation_system.relation_mapping.registry import ( + RelationOperationRegistry, + ) + op = RelationOperationRegistry.get(operation_key) + d = self._authoritative_dialect(node, op) # item 95: _UNRESOLVED | str | None + dialect = getattr(self.backend, "dialect", None) if d is _UNRESOLVED else d + return enrich_materialization( + self.backend, fn, + prefer_operation_keys=frozenset({operation_key}), + dialect=dialect, ) def _bind(self, node: RelationNode, binding: Any) -> Any: diff --git a/tests/relations/test_rel_materialize_residue_enrichment.py b/tests/relations/test_rel_materialize_residue_enrichment.py new file mode 100644 index 00000000..3f1f433c --- /dev/null +++ b/tests/relations/test_rel_materialize_residue_enrichment.py @@ -0,0 +1,163 @@ +"""Relation-subsystem MATERIALIZE_RESIDUE enrichment (item 98). + +The per-op narrowing is dead: _dispatch passes prefer_operation_keys=frozenset() +for handler ops and expression FKEYs (never an RKEY) for declarative ops, so a +dialect-scoped relation residue fact can never fire. This item carries the op's +RKEY into the filter and threads the authoritative dialect into +enrich_materialization. + +Design: mountainash-central +2026-08-14-relation-materialize-residue-enrichment-design.md +(Revision 6, 6 GLM-5.2 adversarial review rounds -- SOUND_WITH_CONCERNS). +""" +from __future__ import annotations + +import pandas as pd +import narwhals as nw +import pytest + +import mountainash as ma +from mountainash.core.capabilities import ( + Boundary, + CapabilityFact, + CapabilityLevel, + CapabilityRegistry, + Enforcement, +) +from mountainash.core.constants import CONST_BACKEND +from mountainash.core.types import BackendCapabilityError +from mountainash.relations.core.relation_system.relation_keys.enums import ( + RKEY_SUBSTRAIT_REL, +) +from mountainash.relations.backends.relation_systems.narwhals.substrait.relsys_nw_set import ( + SubstraitNarwhalsSetRelationSystem, +) + +import mountainash.relations.backends # noqa: F401 +import mountainash.expressions.backends # noqa: F401 + + +class TestDeclarativeUnionResidueFires: + def test_union_all_residue_fact_enriches_forced_native_error(self, monkeypatch): + snap = CapabilityRegistry.snapshot() + try: + CapabilityRegistry.register_backend( + CONST_BACKEND.NARWHALS, + [ + CapabilityFact( + operation_key=RKEY_SUBSTRAIT_REL.UNION_ALL, + param="*", + level=CapabilityLevel.UNSUPPORTED, + backend=CONST_BACKEND.NARWHALS, + dialect="narwhals-pandas", + enforcement=Enforcement.MATERIALIZE_RESIDUE, + boundary=Boundary.MATERIALIZE, + native_errors=(TypeError,), + message="union_all residue fired (test)", + since="2026-08-14", + ) + ], + ) + + def _boom(self, relations): + raise TypeError("forced union_all failure") + + monkeypatch.setattr( + SubstraitNarwhalsSetRelationSystem, "union_all", _boom + ) + + nw_df = nw.from_native(pd.DataFrame({"a": [1]}), eager_only=True) + with pytest.raises(BackendCapabilityError, match="union_all residue fired"): + ma.concat([ma.relation(nw_df), ma.relation(nw_df)]).to_polars() + finally: + CapabilityRegistry.restore(snap) + + +class TestDeadDeclarationEnforcement: + def test_handler_routed_residue_fact_requires_wraps_native_call(self): + from mountainash.relations.core.relation_system.relation_keys.enums import ( + RKEY_MOUNTAINASH_REL, + ) + snap = CapabilityRegistry.snapshot() + try: + with pytest.raises(ValueError, match="wraps_native_call"): + CapabilityRegistry.register_backend( + CONST_BACKEND.NARWHALS, + [ + CapabilityFact( + operation_key=RKEY_MOUNTAINASH_REL.REF, + param="*", + level=CapabilityLevel.UNSUPPORTED, + backend=CONST_BACKEND.NARWHALS, + enforcement=Enforcement.MATERIALIZE_RESIDUE, + boundary=Boundary.MATERIALIZE, + native_errors=(TypeError,), + since="2026-08-14", + ) + ], + ) + finally: + CapabilityRegistry.restore(snap) + + +class TestHandlerPathAndBoundaries: + def test_handler_join_residue_fires(self, monkeypatch): + from mountainash.relations.backends.relation_systems.narwhals.substrait.relsys_nw_join import ( + SubstraitNarwhalsJoinRelationSystem, + ) + snap = CapabilityRegistry.snapshot() + try: + CapabilityRegistry.register_backend( + CONST_BACKEND.NARWHALS, + [ + CapabilityFact( + operation_key=RKEY_SUBSTRAIT_REL.JOIN, param="*", + level=CapabilityLevel.UNSUPPORTED, + backend=CONST_BACKEND.NARWHALS, dialect="narwhals-pandas", + enforcement=Enforcement.MATERIALIZE_RESIDUE, + boundary=Boundary.MATERIALIZE, native_errors=(TypeError,), + message="join residue fired (test)", + since="2026-08-14", + ) + ], + ) + def _boom(self, *a, **k): + raise TypeError("forced join failure") + monkeypatch.setattr(SubstraitNarwhalsJoinRelationSystem, "join", _boom) + nw_df = nw.from_native(pd.DataFrame({"id": [1]}), eager_only=True) + with pytest.raises(BackendCapabilityError, match="join residue fired"): + ma.relation(nw_df).join(ma.relation(nw_df), on="id").to_polars() + finally: + CapabilityRegistry.restore(snap) + + def test_child_visit_error_not_narrowed(self, monkeypatch): + from mountainash.relations.core.unified_visitor import relation_visitor as rv + snap = CapabilityRegistry.snapshot() + try: + CapabilityRegistry.register_backend( + CONST_BACKEND.NARWHALS, + [ + CapabilityFact( + operation_key=RKEY_SUBSTRAIT_REL.JOIN, param="*", + level=CapabilityLevel.UNSUPPORTED, + backend=CONST_BACKEND.NARWHALS, dialect="narwhals-pandas", + enforcement=Enforcement.MATERIALIZE_RESIDUE, + boundary=Boundary.MATERIALIZE, native_errors=(TypeError,), + message="join residue fired (test)", + since="2026-08-14", + ) + ], + ) + # Force the RIGHT-side child visit/coercion to raise BEFORE the + # join's native call -- the join's (JOIN, *) fact must NOT enrich + # this child-visit error (children compile outside the wrap). + monkeypatch.setattr( + rv.UnifiedRelationVisitor, + "_visit_and_coerce_right", + lambda self, right, left: (_ for _ in ()).throw(TypeError("child visit failure")), + ) + nw_df = nw.from_native(pd.DataFrame({"id": [1]}), eager_only=True) + with pytest.raises(TypeError, match="child visit failure"): + ma.relation(nw_df).join(ma.relation(nw_df), on="id").to_polars() + finally: + CapabilityRegistry.restore(snap) diff --git a/tests/relations/test_rel_visitor.py b/tests/relations/test_rel_visitor.py index 8023e6ce..d0e8d458 100644 --- a/tests/relations/test_rel_visitor.py +++ b/tests/relations/test_rel_visitor.py @@ -532,7 +532,7 @@ def test_present_expression_function_keys_filters_to_expression_args(self): ) from mountainash.relations.core.relation_nodes import ProjectRelNode from mountainash.relations.core.unified_visitor.relation_visitor import ( - _present_expression_function_keys, + _present_operation_keys, ) op = RelationOperationRegistry.get(RKEY_SUBSTRAIT_REL.PROJECT_SELECT) @@ -544,8 +544,8 @@ def test_present_expression_function_keys_filters_to_expression_args(self): ], operation=RKEY_SUBSTRAIT_REL.PROJECT_SELECT, ) - assert _present_expression_function_keys(node, op) == frozenset( - {FK_STR.SPLIT, FK_LIST.CONTAINS} + assert _present_operation_keys(node, op) == frozenset( + {FK_STR.SPLIT, FK_LIST.CONTAINS, RKEY_SUBSTRAIT_REL.PROJECT_SELECT} ) def test_no_cycle_recursion_error_on_self_referential_options(self):