Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 0 additions & 13 deletions docs/reference/expression-coverage.json
Original file line number Diff line number Diff line change
Expand Up @@ -70849,19 +70849,6 @@
"upstream_ref": null,
"since": "2026-08-06"
},
{
"id": "NW-REL-03",
"kind": "engine_leniency",
"operation_keys": [],
"backends": [
"narwhals-lazy"
],
"summary": "narwhals-lazy has no sample() on a LazyFrame (AttributeError)",
"impact": "Relation.sample() raises on narwhals-lazy; eager backends sample rows",
"workaround": "Use an eager backend for sample()",
"upstream_ref": null,
"since": "2026-08-06"
},
{
"id": "NW-STR-14",
"kind": "semantics",
Expand Down
1 change: 0 additions & 1 deletion docs/reference/expression-coverage.md
Original file line number Diff line number Diff line change
Expand Up @@ -578,7 +578,6 @@ Cells whose facts are all scoped (dialect / parameter / option / value-class) ha
| NW-MATH-10 | engine_leniency | pandas, narwhals | `SIN`, `COS`, `TAN`, `ASIN`, `ACOS`, `ATAN`, `ATAN2`, `RADIANS`, `DEGREES`, `SINH`, `COSH`, `TANH`, `ASINH`, `ACOSH`, `ATANH` | pandas and narwhals lack native trigonometric, angular-conversion, and hyperbolic math functions; these ops raise NotImplementedError | trig (sin/cos/tan/asin/acos/atan/atan2), angular (radians/degrees), and hyperbolic (sinh/cosh/tanh/asinh/acosh/atanh) raise on pandas and all narwhals backends; polars and ibis (polars/duckdb) compute them | Use a polars or ibis-polars/ibis-duckdb backend for these math functions | NW-MATH-10 | 2026-08-06 |
| NW-REL-01 | engine_leniency | narwhals-lazy | — | narwhals-lazy with_row_index() requires an explicit order_by= (row order over a LazyFrame is undefined); calling it without one raises TypeError | Relation.with_row_index() raises on narwhals-lazy; eager narwhals/polars and ibis-duckdb/ibis-sqlite assign a 0..N-1 index | Use an eager backend, or pass an explicit order before the lazy row index | — | 2026-08-06 |
| NW-REL-02 | engine_leniency | narwhals | — | Narwhals does not support unnest of a struct column | Relation.unnest() raises on narwhals backends; polars and ibis compute it | Use a polars or ibis backend for unnest | — | 2026-08-06 |
| NW-REL-03 | engine_leniency | narwhals-lazy | — | narwhals-lazy has no sample() on a LazyFrame (AttributeError) | Relation.sample() raises on narwhals-lazy; eager backends sample rows | Use an eager backend for sample() | — | 2026-08-06 |
| NW-STR-14 | semantics | narwhals-pandas | `TITLE`, `INITCAP` | narwhals-pandas title/initcap route to pandas str.title(); its Unicode titlecasing of sharp-S/ligatures differs from polars to_titlecase (e.g. 'ße' -> 'ẞe' vs 'SSe') | title()/initcap() on narwhals-pandas may differ from polars/narwhals-polars on non-ASCII inputs (sharp-S, ligatures); ASCII is identical | Use polars or narwhals-polars where exact polars titlecasing of non-ASCII is required | — | 2026-07-29 |
| NW-STR-15 | semantics | pandas, narwhals | `LTRIM`, `RTRIM` | pandas and narwhals lack directional trimming: ltrim/rtrim and strip_chars_start/end strip BOTH sides (only strip_chars is native), so leading/trailing-only requests over-strip | ltrim/rtrim and str.strip_chars_start()/strip_chars_end() strip both sides on pandas/narwhals; polars and ibis strip only the requested side | Use a polars or ibis backend for directional trimming | — | 2026-08-06 |
| NW-STR-17 | engine_leniency | pandas, narwhals | `REPEAT` | str.repeat(n) is unsupported on pandas and narwhals (BackendCapabilityError); no repeat translation is wired for these backends | ma.col(x).str.repeat(n) raises on pandas and all narwhals backends; polars and ibis compute it | Use a polars or ibis backend for str.repeat() | — | 2026-08-06 |
Expand Down
11 changes: 0 additions & 11 deletions src/mountainash/core/capabilities/divergences.py
Original file line number Diff line number Diff line change
Expand Up @@ -647,17 +647,6 @@ def _all() -> tuple[DivergenceFact, ...]:
upstream_ref=None,
since="2026-08-06",
),
DivergenceFact(
id="NW-REL-03",
kind=DivergenceKind.ENGINE_LENIENCY,
operation_keys=(), # relation op sample
backends=("narwhals-lazy",),
summary="narwhals-lazy has no sample() on a LazyFrame (AttributeError)",
impact="Relation.sample() raises on narwhals-lazy; eager backends sample rows",
workaround="Use an eager backend for sample()",
upstream_ref=None,
since="2026-08-06",
),
DivergenceFact(
id="IB-REL-10",
kind=DivergenceKind.ENGINE_LENIENCY,
Expand Down
7 changes: 1 addition & 6 deletions src/mountainash/datacontracts/compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,16 +109,11 @@ def constraint_checks(
)
)
if c is not None and c.pattern is not None:
# `pattern` is a regex (Frictionless pattern / regex-match semantics),
# NOT a literal substring — str.contains is Substrait literal contains,
# so a regex must go through regexp_match_substring (partial match ->
# not-null == the pattern matched somewhere).
checks.append(
RowRule(
id=f"{col}__pattern",
expr=_maybe_guard(
nullable, col,
ma.col(col).str.regexp_match_substring(c.pattern).is_not_null(),
nullable, col, ma.col(col).str.regex_contains(c.pattern)
),
severity=severity,
fields=[col],
Expand Down
48 changes: 30 additions & 18 deletions src/mountainash/datacontracts/validator.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from __future__ import annotations

import inspect
import random
from typing import TYPE_CHECKING, Any, Callable

import polars as pl
Expand Down Expand Up @@ -60,31 +61,38 @@ def _to_polars_frame(rel: Any) -> pl.DataFrame:
if isinstance(materialised, pl.LazyFrame):
materialised = materialised.collect()
return materialised

@classmethod
def _slice(
cls, rel: Any, *, head: int | None, tail: int | None,
sample: int | None, random_seed: int | None,
) -> Any:
"""17 P4: slicing happens exactly once, before the runner."""
import mountainash as ma

cls,
rel: Any,
*,
head: int | None,
tail: int | None,
sample: int | None,
random_seed: int | None,
) -> tuple[Any, dict[str, Any]]:
"""Apply slicing once, keeping seeded sampling on the source backend."""
diagnostics: dict[str, Any] = {}
if head is not None:
rel = rel.head(head)
if tail is not None:
rel = rel.tail(tail)
if sample is not None:
if random_seed is None:
# Relation.sample is cross-backend (no seed param; Ibis
# approximates n via fraction) — stays on the native backend.
rel = rel.sample(n=sample)
effective_seed = (
random_seed if random_seed is not None else random.randrange(2**31)
)
sampled = rel.sample(n=sample, seed=effective_seed)
if sampled.count_rows() == 0 and rel.count_rows() > 0:
rel = rel.limit(sample)
diagnostics["sample_fallback"] = {
"reason": "sampled slice was empty on non-empty input",
"requested_sample": sample,
"random_seed": effective_seed,
"fallback": f"limit({sample})",
}
else:
# Seeded sampling: Relation.sample has no seed parameter, so
# deterministic sampling materialises to Polars (documented
# narrowing; follow-on backlog: seed option on Relation.sample).
frame = cls._to_polars_frame(rel)
rel = ma.relation(frame.sample(n=sample, seed=random_seed))
return rel
rel = sampled
return rel, diagnostics

# -- public API -----------------------------------------------------------

Expand Down Expand Up @@ -175,7 +183,9 @@ def _run(
# --- data phase
prepared = self._prepare_data(data, context)
rel = prepared if isinstance(prepared, Relation) else ma.relation(prepared)
rel = self._slice(rel, head=head, tail=tail, sample=sample, random_seed=random_seed)
rel, slice_diagnostics = self._slice(
rel, head=head, tail=tail, sample=sample, random_seed=random_seed
)

if getattr(self.contract.Config, "coerce", True):
rel = rel.conform(spec)
Expand All @@ -191,6 +201,8 @@ def _run(
validator_name=self.name,
datacontract_name=self.contract.contract_name(),
)
if slice_diagnostics:
result.diagnostics.update(slice_diagnostics)
if skipped:
# skipped summaries are visibility only: appended to the frame,
# never part of the runner's pass computation (they cannot fail)
Expand Down
2 changes: 2 additions & 0 deletions src/mountainash/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
ConformTransformError,
SchemaDriftError,
)
from mountainash.relations.core.errors import InvalidSampleArgumentsError
from mountainash.relations.dag.errors import (
DAGError,
RelationDAGRequired,
Expand Down Expand Up @@ -53,6 +54,7 @@
__all__ = [
"MountainashError",
"InvalidOptionValueError",
"InvalidSampleArgumentsError",
"BareExpressionCollectionError",
"ConformError",
"MissingFieldsError",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,14 +45,20 @@ def explode(self, relation: ir.Table, /, *, columns: list[str]) -> ir.Table:
return result

def sample(
self, relation: ir.Table, /, *, n: Optional[int] = None, fraction: Optional[float] = None
self,
relation: ir.Table,
/,
*,
n: Optional[int] = None,
fraction: Optional[float] = None,
seed: Optional[int] = None,
) -> ir.Table:
if fraction is not None:
return relation.sample(fraction)
return relation.sample(fraction, method="row", seed=seed)
if n is not None:
total = relation.count().execute()
frac = min(n / total, 1.0) if total > 0 else 1.0
return relation.sample(frac)
return relation.sample(frac, method="row", seed=seed)
raise ValueError("Either n or fraction must be specified for sample().")

def unpivot(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,16 @@ def sample(
*,
n: Optional[int] = None,
fraction: Optional[float] = None,
seed: Optional[int] = None,
) -> Any:
return relation.sample(n=n, fraction=fraction)
frame = relation
is_lazy = isinstance(frame, nw.LazyFrame)
if is_lazy:
frame = frame.collect()
if n is not None:
n = min(n, len(frame))
sampled = frame.sample(n=n, fraction=fraction, seed=seed)
return sampled.lazy() if is_lazy else sampled

def unpivot(
self,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,13 @@ def sample(
*,
n: Optional[int] = None,
fraction: Optional[float] = None,
seed: Optional[int] = None,
) -> pl.LazyFrame:
# LazyFrame does not support .sample() directly — collect, sample, re-lazy.
return relation.collect().sample(n=n, fraction=fraction).lazy()
frame = relation.collect()
if n is not None:
n = min(n, frame.height)
return frame.sample(n=n, fraction=fraction, seed=seed).lazy()

def unpivot(
self,
Expand Down
4 changes: 4 additions & 0 deletions src/mountainash/relations/core/errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,7 @@ def __init__(self, node_type: type) -> None:
f"via RelationVisitRegistry.register({node_type.__name__}, handler) "
f"or define _operation_key plus a RelationOperationDef for it."
)


class InvalidSampleArgumentsError(MountainashError, ValueError):
"""Relation.sample() argument-contract violation."""
18 changes: 17 additions & 1 deletion src/mountainash/relations/core/relation_api/relation.py
Original file line number Diff line number Diff line change
Expand Up @@ -407,13 +407,29 @@ def sample(
*,
n: Optional[int] = None,
fraction: Optional[float] = None,
seed: Optional[int] = None,
) -> Relation:
"""Sample rows."""
"""Sample rows using a validated common argument contract."""
from mountainash.relations.core.errors import InvalidSampleArgumentsError

if (n is None) == (fraction is None):
raise InvalidSampleArgumentsError(
"sample() requires exactly one of n or fraction "
f"(got n={n!r}, fraction={fraction!r})"
)
if n is not None and n < 0:
raise InvalidSampleArgumentsError(f"sample(n=...) must be >= 0, got {n}")
if fraction is not None and not (0.0 <= fraction <= 1.0):
raise InvalidSampleArgumentsError(
f"sample(fraction=...) must be in [0, 1], got {fraction}"
)
options: dict[str, Any] = {}
if n is not None:
options["n"] = n
if fraction is not None:
options["fraction"] = fraction
if seed is not None:
options["seed"] = seed
return self._make(
ExtensionRelNode(
input=self._node,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,13 @@ def drop_nans(self, *, subset: Optional[list[str]] = None) -> Self: ...
def with_row_index(self, *, name: str = "index") -> Self: ...
def explode(self, *columns: Any) -> Self: ...
def unnest(self, *columns: str, separator: str) -> Self: ...
def sample(self, *, n: Optional[int] = None, fraction: Optional[float] = None) -> Self: ...
def sample(
self,
*,
n: Optional[int] = None,
fraction: Optional[float] = None,
seed: Optional[int] = None,
) -> Self: ...

def unpivot(
self,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,13 @@ def with_row_index(self, relation: RelationT, /, *, name: str = "index") -> Rela
def explode(self, relation: RelationT, /, *, columns: list[str]) -> RelationT: ...

def sample(
self, relation: RelationT, /, *, n: Optional[int] = None, fraction: Optional[float] = None
self,
relation: RelationT,
/,
*,
n: Optional[int] = None,
fraction: Optional[float] = None,
seed: Optional[int] = None,
) -> RelationT: ...

def unpivot(
Expand Down
1 change: 0 additions & 1 deletion tests/_spine_expectation_census.md
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,6 @@ Buckets: `migrated` (derivable from the spine today), `retained` (a LITERAL_ONLY
| tests/relations/cross_backend/test_rel_extension_ops_results.py:39 | static-marker | UNRESOLVED | UNRESOLVED | UNRESOLVED | None | spine-derived id-keyed divergence mark via xfail_divergence('NW-REL-01') — migrated |
| tests/relations/cross_backend/test_rel_extension_ops_results.py:40 | static-marker | UNRESOLVED | UNRESOLVED | UNRESOLVED | None | spine-derived id-keyed divergence mark via xfail_divergence('NW-REL-02') — migrated |
| tests/relations/cross_backend/test_rel_extension_ops_results.py:41 | static-marker | UNRESOLVED | UNRESOLVED | UNRESOLVED | None | spine-derived id-keyed divergence mark via xfail_divergence('MA-REL-01') — migrated |
| tests/relations/cross_backend/test_rel_extension_ops_results.py:42 | static-marker | UNRESOLVED | UNRESOLVED | UNRESOLVED | None | spine-derived id-keyed divergence mark via xfail_divergence('NW-REL-03') — migrated |
| tests/relations/cross_backend/test_rel_join_results.py:42 | static-marker | UNRESOLVED | UNRESOLVED | UNRESOLVED | None | spine-derived id-keyed divergence mark via xfail_divergence('IB-REL-11') — migrated |
| tests/relations/cross_backend/test_terminal_scalar_aggregates.py:13 | static-marker | UNRESOLVED | UNRESOLVED | UNRESOLVED | None | spine-derived id-keyed divergence mark via xfail_divergence('IB-AGG-05') — migrated |
| tests/relations/cross_backend/test_terminal_scalar_aggregates.py:14 | static-marker | UNRESOLVED | UNRESOLVED | UNRESOLVED | None | spine-derived id-keyed divergence mark via xfail_divergence('NW-AGG-03') — migrated |
Expand Down
40 changes: 40 additions & 0 deletions tests/datacontracts/test_compiler.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""Tests for contract_from_typespec — TypeSpec to native BaseDataContract."""
from __future__ import annotations

import pytest
import polars as pl

from mountainash.typespec.spec import TypeSpec, FieldSpec, FieldConstraints
Expand Down Expand Up @@ -208,3 +209,42 @@ def test_no_constraints_produces_nullable_field(self):
df = pl.DataFrame({"val": [None, "x"]})
result = Contract.validate_datacontract(df)
assert result.passes is True

class TestPatternCheckCrossBackend:
"""Pattern checks must fail on non-matching values on every backend."""

def _spec(self):
return _make_spec(
FieldSpec(
name="code",
type=UniversalType.STRING,
constraints=FieldConstraints(pattern=r"^[a-z]{3}-[0-9]{2}$"),
)
)

def _data(self, backend, values):
df = pl.DataFrame({"code": values})
if backend == "narwhals-pandas":
import narwhals as nw

return nw.from_native(df.to_pandas())
if backend == "ibis-duckdb":
ibis = pytest.importorskip("ibis")

return ibis.duckdb.connect().create_table("t", df.to_arrow())
return df

@pytest.mark.parametrize("backend", ["polars", "narwhals-pandas", "ibis-duckdb"])
def test_non_matching_value_fails(self, backend):
result = self._spec().to_contract(name="pattern_xb").validate_datacontract(
self._data(backend, ["abc-12", "###"])
)
assert not result.passes
assert "code__pattern" in _failing_check_ids(result)

@pytest.mark.parametrize("backend", ["polars", "narwhals-pandas", "ibis-duckdb"])
def test_matching_values_pass(self, backend):
result = self._spec().to_contract(name="pattern_xb").validate_datacontract(
self._data(backend, ["abc-12", "xyz-99"])
)
assert result.passes
Loading
Loading