From cec18a9f276d0d04061c8b5ce4f779b08c7b179f Mon Sep 17 00:00:00 2001 From: Nathaniel Ramm Date: Sat, 15 Aug 2026 02:26:55 +1000 Subject: [PATCH] feat(relations): coerce bare cross-family DAG dependency refs (item 92) A DAG target anchored on family X that joins a BARE foreign-family dependency ref now compiles the ref in its own family and coerces the result to the anchor's family via _coerce_to_match (item 94), instead of leaving the ref on the anchor pair to fail in read(). - _anchor_prototype(family, dialect) + _is_lazy_narwhals helpers - _resolve_identity_leaf(name) -> (family, dialect, ReadRelNode) - foreign branch: bare ReadRelNode -> compile own family + coerce; derived root stays on the anchor pair (item 97's territory) - upfront lazy-narwhals-anchor guard (order-independent rejection) Design: mountainash-central 2026-08-14-dag-cross-family-bare-dependency-coercion.md (Rev 5) --- src/mountainash/relations/dag/dag.py | 95 +++++++++++++++++-- .../dag/test_dag_cross_family_bare.py | 61 ++++++++++++ .../dag/test_dag_dialect_identity.py | 25 ++--- 3 files changed, 158 insertions(+), 23 deletions(-) create mode 100644 tests/relations/dag/test_dag_cross_family_bare.py diff --git a/src/mountainash/relations/dag/dag.py b/src/mountainash/relations/dag/dag.py index 63d78c59..ae29eef1 100644 --- a/src/mountainash/relations/dag/dag.py +++ b/src/mountainash/relations/dag/dag.py @@ -4,9 +4,40 @@ from typing import Any, Optional, TYPE_CHECKING from mountainash.core.constants import CONST_BACKEND +from mountainash.relations.core.relation_nodes import ReadRelNode from mountainash.relations.core.relation_nodes.extensions_mountainash import RefRelNode from mountainash.relations.dag.traversal import walk_refs as _walk_refs + +def _anchor_prototype(family: CONST_BACKEND, dialect: str | None) -> Any: + """A lightweight empty object of *family* for coercion to target.""" + if family is CONST_BACKEND.POLARS: + import polars as pl + return pl.DataFrame({}).lazy() + if family is CONST_BACKEND.IBIS: + import ibis + return ibis.memtable({}) + if family is CONST_BACKEND.NARWHALS: + import narwhals as nw + if dialect == "narwhals-polars": + import polars as pl + return nw.from_native(pl.DataFrame({}), eager_only=True) + if dialect == "narwhals-pyarrow": + import pyarrow as pa + return nw.from_native(pa.table({}), eager_only=True) + import pandas as pd + return nw.from_native(pd.DataFrame({}), eager_only=True) + return None + + +def _is_lazy_narwhals(obj: Any) -> bool: + """True iff *obj* is a narwhals LazyFrame.""" + try: + import narwhals as nw + return isinstance(obj, nw.LazyFrame) + except Exception: + return False + if TYPE_CHECKING: from mountainash.conform.drift import ConformCollection from mountainash.core.dtypes import MountainashDtype @@ -367,6 +398,20 @@ def _compile_with_refs( expression_system = expression_system_cls(dialect=dialect) expr_visitor = UnifiedExpressionVisitor(expression_system) + if backend_target_name is not None or ref_names: + anchor_name = backend_target_name or sorted(ref_names)[0] + anchor_family, _, anchor_leaf = self._resolve_identity_leaf(anchor_name) + if anchor_leaf is not None and _is_lazy_narwhals(anchor_leaf.dataframe): + if any( + self._resolve_actual_identity_for(n)[0] + not in (None, resolved_backend) + and isinstance(getattr(self.relations[n], "_node", None), ReadRelNode) + for n in ref_names + ): + raise TypeError( + "Cross-family DAG coercion is not supported with a lazy Narwhals anchor." + ) + cache: dict[str, Any] = {} def resolver(n: str) -> Any: @@ -448,14 +493,24 @@ def resolver(n: str) -> Any: expr_visitor, ) elif ref_family != resolved_backend: - # Genuinely different family -- item 92's territory (no - # cross-family coercion attempted here). Leave this ref - # on the anchor pair; NEVER construct an invalid - # (family, dialect) hybrid for a foreign family. - visitor.backend, visitor.expr_visitor = ( - relation_system, - expr_visitor, - ) + if isinstance(root, ReadRelNode): + anchor_proto = _anchor_prototype(resolved_backend, dialect) + visitor.backend = get_relation_system(ref_family)(dialect=ref_dialect) + visitor.expr_visitor = UnifiedExpressionVisitor( + get_expression_system(ref_family)(dialect=ref_dialect) + ) + visitor.key_context = KeyDriftContext( + resource_name=n, + constraints_for=self.constraints_for, + schema_of=self.schema, + ) + result = root.accept(visitor) + cache[n] = UnifiedRelationVisitor._coerce_to_match( + anchor_proto, result + ) + visitor.backend, visitor.expr_visitor = relation_system, expr_visitor + continue + visitor.backend, visitor.expr_visitor = relation_system, expr_visitor elif ref_dialect != dialect: # Same family, dialect differs from the anchor's -- # covers BOTH a genuinely different known dialect AND @@ -653,6 +708,30 @@ def _resolve_actual_identity_for( pass return None, None + def _resolve_identity_leaf( + self, target_name: str + ) -> "tuple[CONST_BACKEND | None, str | None, Any | None]": + """Return the identity and leaf selected for a named relation.""" + from mountainash.relations.core.relation_api.relation_base import RelationBase + from mountainash.core.backend_detection import identify_backend_identity + from mountainash.relations.dag.errors import RelationDAGRequired + + for n in self.topological_order(target=target_name): + root = getattr(self.relations[n], "_node", None) + if root is None: + continue + try: + read_node = RelationBase._find_leaf_read_node(root) + except (ValueError, AttributeError, RelationDAGRequired): + continue + if read_node is not None: + try: + identity = identify_backend_identity(read_node.dataframe) + return identity.family, identity.dialect, read_node + except Exception: + pass + return None, None, None + def _resolve_actual_identity_for_node( self, node: Any ) -> "tuple[CONST_BACKEND | None, str | None]": diff --git a/tests/relations/dag/test_dag_cross_family_bare.py b/tests/relations/dag/test_dag_cross_family_bare.py new file mode 100644 index 00000000..4a1c3451 --- /dev/null +++ b/tests/relations/dag/test_dag_cross_family_bare.py @@ -0,0 +1,61 @@ +"""Cross-family coercion for bare DAG dependency refs (item 92).""" +from __future__ import annotations + +import pandas as pd +import polars as pl +import pytest + +import mountainash as ma +from mountainash.relations.dag import RelationDAG + +import mountainash.relations.backends # noqa: F401 +import mountainash.expressions.backends # noqa: F401 + + +class TestBareForeignDependencyCoercion: + def test_polars_anchor_joins_bare_pandas_dependency(self): + dag = RelationDAG() + dag.add("a_anchor", ma.relation(pl.DataFrame({"id": [1, 2], "name": ["a", "b"]}))) + dag.add("z_dep", ma.relation(pd.DataFrame({"id": [2, 3], "name": ["c", "d"]}))) + dag.add("target", dag.ref("a_anchor").join(dag.ref("z_dep"), on="id")) + + result = dag.collect("target") + + assert result.collect().to_dict(as_series=False) == { + "id": [2], + "name": ["b"], + "name_right": ["c"], + } + +class TestCoercionMatrixAndBoundaries: + def test_ibis_anchor_coerces_bare_pandas_dependency(self): + import ibis + + ib = ibis.memtable(pl.DataFrame({"id": [1, 2], "name": ["a", "b"]})) + dag = RelationDAG() + dag.add("a_anchor", ma.relation(ib)) + dag.add("z_dep", ma.relation(pd.DataFrame({"id": [2], "name": ["c"]}))) + dag.add("target", dag.ref("a_anchor").join(dag.ref("z_dep"), on="id")) + result = dag.collect("target") + assert result is not None + + def test_narwhals_anchor_coerces_bare_polars_dependency_to_exact_dialect(self): + import narwhals as nw + + nw_pd = nw.from_native(pd.DataFrame({"id": [1, 2]}), eager_only=True) + dag = RelationDAG() + dag.add("a_anchor", ma.relation(nw_pd)) # narwhals-pandas + dag.add("z_dep", ma.relation(pl.DataFrame({"id": [2]}))) + dag.add("target", dag.ref("a_anchor").join(dag.ref("z_dep"), on="id")) + result = dag.collect("target") + assert result is not None + + def test_derived_foreign_dependency_is_not_coerced(self): + # Boundary (item 97): a FilterRelNode-rooted foreign dependency stays on + # the anchor pair and raises the raw TypeError. + dag = RelationDAG() + dag.add("a_anchor", ma.relation(pl.DataFrame({"id": [1]}))) + dag.add("z_dep", ma.relation(pd.DataFrame({"id": [2]})).filter(ma.col("id") > 0)) + dag.add("target", dag.ref("a_anchor").join(dag.ref("z_dep"), on="id")) + with pytest.raises(TypeError, match="cannot read DataFrame"): + dag.collect("target") diff --git a/tests/relations/dag/test_dag_dialect_identity.py b/tests/relations/dag/test_dag_dialect_identity.py index 898a4955..e55843c0 100644 --- a/tests/relations/dag/test_dag_dialect_identity.py +++ b/tests/relations/dag/test_dag_dialect_identity.py @@ -364,11 +364,9 @@ class TestExplicitBackendPerRefNeverBuildsInvalidHybrid: """Round-1 finding (per-ref level, distinct from Task 3's anchor-level fix): an explicit backend= compile call must not construct an invalid hybrid for a NON-anchor ref whose OWN physical family differs from the - override. Uses a real Polars-native anchor (so the override is not - trivially "the same as detection") plus a separate Narwhals-pandas - ref, and asserts on the non-anchor ref specifically -- discriminating - Task 4's `ref_family != resolved_backend` branch, not just Task 3's - anchor-coherence fix.""" + override. Item 92 changed the resolution: a BARE foreign ref is now + compiled in its OWN family (then coerced to the override family), so the + ref's visit entry observes its own family, never an invalid hybrid.""" def test_pandas_ref_under_explicit_polars_backend_with_polars_anchor( self, _dialect_spy_factory @@ -384,18 +382,15 @@ def test_pandas_ref_under_explicit_polars_backend_with_polars_anchor( ) _dialect_spy_factory(pandas_rel._node, "pandas_ref") - try: - dag.collect("final", backend="polars") - except Exception: - pass # PolarsRelationSystem cannot read a raw Narwhals frame -- - # this ref is on a genuinely different family than the - # override (item 92's territory), so its OWN read is - # expected to fail; we only assert the STATE at its visit() - # entry, captured before that failure. + # Item 92: the bare narwhals-pandas ref is compiled in its own family + # (narwhals) and coerced to the explicit polars override -- so its + # visit entry observes narwhals, not the invalid polars hybrid. + result = dag.collect("final", backend="polars") + assert result is not None captured = _dialect_spy_factory.captured["pandas_ref"]["entry"] - assert captured["backend_type"] == CONST_BACKEND.POLARS - assert captured["backend_dialect"] != "narwhals-pandas" + assert captured["backend_type"] == CONST_BACKEND.NARWHALS + assert captured["backend_dialect"] == "narwhals-pandas" class TestSameFamilyUnboundDialectRefGetsNoneNotAnchorsDialect: