diff --git a/src/mountainash/conform/expressions.py b/src/mountainash/conform/expressions.py index 62756f17..ca834f78 100644 --- a/src/mountainash/conform/expressions.py +++ b/src/mountainash/conform/expressions.py @@ -956,12 +956,8 @@ def _build_field_expr( # Other backends fall through to base type cast only. elif fld.categories is not None: # Extract values from categories (handles both simple and object forms) - cat_values: list[Any] = [] - for cat in fld.categories: - if isinstance(cat, dict): - cat_values.append(cat["value"]) - else: - cat_values.append(cat) + from mountainash.typespec._categorical import categorical_values + cat_values = categorical_values(fld.categories) # Step 1: base type cast (if needed) if fld.type and fld.type != UniversalType.ANY: diff --git a/src/mountainash/core/dtypes/target_narwhals.py b/src/mountainash/core/dtypes/target_narwhals.py index fe03e9d5..a0def96b 100644 --- a/src/mountainash/core/dtypes/target_narwhals.py +++ b/src/mountainash/core/dtypes/target_narwhals.py @@ -62,4 +62,8 @@ def parse_type_string(s: str) -> Optional[Any]: "Categorical", "Enum") } result = parse_constructor_repr(s, namespace) - return result if isinstance(result, type) or result is not None else None + # No nw.DataType base class exists to isinstance-check instances against; + # the namespace is closed + AST-validated upstream, so any non-None + # construction is a legitimate parameterized dtype (DTypeClass instances + # are not `type`s — isinstance(None-check) is precisely `result is not None`). + return result diff --git a/src/mountainash/core/dtypes/target_pyarrow.py b/src/mountainash/core/dtypes/target_pyarrow.py index da53ac74..93a92399 100644 --- a/src/mountainash/core/dtypes/target_pyarrow.py +++ b/src/mountainash/core/dtypes/target_pyarrow.py @@ -60,18 +60,24 @@ def parse_type_string(s: str) -> Optional[Any]: return pa.type_for_alias(s) # unparameterized names, unchanged except (KeyError, ValueError): pass - if m := _TIMESTAMP_RE.match(s): - unit, tz = m.groups() - return pa.timestamp(unit, tz=tz) if tz else pa.timestamp(unit) - if m := _DURATION_RE.match(s): - return pa.duration(m.group(1)) - if m := _TIME_RE.match(s): - bits, unit = m.groups() - return (pa.time32 if bits == "32" else pa.time64)(unit) - if m := _DECIMAL_RE.match(s): - bits, prec, scale = m.groups() - ctor = pa.decimal128 if bits == "128" else pa.decimal256 - return ctor(int(prec), int(scale)) + try: + if m := _TIMESTAMP_RE.match(s): + unit, tz = m.groups() + return pa.timestamp(unit, tz=tz) if tz else pa.timestamp(unit) + if m := _DURATION_RE.match(s): + return pa.duration(m.group(1)) + if m := _TIME_RE.match(s): + bits, unit = m.groups() + return (pa.time32 if bits == "32" else pa.time64)(unit) + if m := _DECIMAL_RE.match(s): + bits, prec, scale = m.groups() + ctor = pa.decimal128 if bits == "128" else pa.decimal256 + return ctor(int(prec), int(scale)) + except (KeyError, ValueError): + # Syntactically-valid-but-semantically-invalid (timestamp[badunit], + # decimal128(999, 10)) — return None so the resolver raises the typed + # InvalidBackendTypeError instead of leaking a raw Arrow ValueError. + return None return None diff --git a/src/mountainash/typespec/_categorical.py b/src/mountainash/typespec/_categorical.py new file mode 100644 index 00000000..dcf32fd2 --- /dev/null +++ b/src/mountainash/typespec/_categorical.py @@ -0,0 +1,20 @@ +"""Shared categorical value extraction (item 54, gap 3). + +The value-extraction logic for Frictionless ``categories`` (simple +``["a", "b"]`` form vs object ``[{"value":.., "label":..}]`` form) was +copy-paste-identical between conform's stage-5b expression builder and the +typespec converters. This module is the single home for it — both call sites +import from here so they can never diverge. +""" +from __future__ import annotations + +from typing import Any + + +def categorical_values(categories: list[Any]) -> list[Any]: + """Extract raw values from a categories list (simple or {value,label} + object form). Returns a NEW list — never aliases the input.""" + return [c["value"] if isinstance(c, dict) else c for c in categories] + + +__all__ = ["categorical_values"] diff --git a/src/mountainash/typespec/converters.py b/src/mountainash/typespec/converters.py index 40a66684..560e6089 100644 --- a/src/mountainash/typespec/converters.py +++ b/src/mountainash/typespec/converters.py @@ -17,6 +17,7 @@ InvalidBackendTypeError, MountainashDtype, TypeTarget, + UnknownDtypeError, registry, ) from mountainash.typespec.universal_types import to_canonical @@ -48,7 +49,51 @@ def _resolve_field_native(field: "FieldSpec", target: TypeTarget) -> Any: canon = to_canonical(field.type) if canon is None: # ANY canon = MountainashDtype.STRING - return registry.to_native_schema(canon, target) + native = registry.to_native_schema(canon, target) + + if canon is MountainashDtype.LIST and field.item_type: + native = _resolve_list_inner(field.name, field.item_type, target, native) + return native + + +def _resolve_list_inner( + field_name: str, item_type_str: str, target: TypeTarget, bare_native: Any +) -> Any: + """Parameterize a bare list container with its Frictionless item_type + (item 54, gap 2). Layered AFTER the backend_type/raise branch — a + backend_type wins first; canonical LIST + item_type is the fallback + enrichment, not a new top-priority branch. + + Pandas returns bare_native unchanged — no native parameterized list + dtype, "object" is the correct, only representation. + """ + from mountainash.typespec.universal_types import parse_universal + try: + item_universal = parse_universal(item_type_str) + except UnknownDtypeError as e: + # parse_universal raises without field context; chain it so the error + # is traceable to its source field while keeping UnknownDtypeError + # (the repo convention for "input not recognized as any dtype"). + raise UnknownDtypeError( + f"field {field_name!r}: item_type {item_type_str!r} is not a " + f"recognized UniversalType" + ) from e + item_canon = to_canonical(item_universal) + if item_canon is None: # item_type == "any" — no parameterization possible + return bare_native + inner_native = registry.to_native_schema(item_canon, target) + if target is TypeTarget.POLARS: + from mountainash.core.lazy_imports import import_polars + return import_polars().List(inner_native) + if target is TypeTarget.NARWHALS: + from mountainash.core.lazy_imports import import_narwhals + return import_narwhals().List(inner_native) + if target is TypeTarget.PYARROW: + from mountainash.core.lazy_imports import import_pyarrow + return import_pyarrow().list_(inner_native) + if target is TypeTarget.IBIS: + return f"array<{inner_native}>" # ibis schema is string-keyed + return bare_native # PANDAS — no native parameterized list dtype # ============================================================================ @@ -75,25 +120,43 @@ def to_polars_schema(schema: TypeSpec) -> Dict[str, Any]: {'id': Int64, 'name': Utf8} """ from mountainash.core.lazy_imports import import_polars + from mountainash.typespec._categorical import categorical_values pl = import_polars() if pl is None: raise ImportError("polars is required for to_polars_schema()") - return {f.name: _resolve_field_native(f, TypeTarget.POLARS) for f in schema.fields} + result = {} + for f in schema.fields: + if f.categories is not None: + # categories takes priority over backend_type/type entirely + # (mirrors conform stage 5's branch order exactly). + values = categorical_values(f.categories) + result[f.name] = ( + pl.Enum([str(v) for v in values]) if f.categories_ordered + else pl.Categorical + ) + else: + result[f.name] = _resolve_field_native(f, TypeTarget.POLARS) + return result # ============================================================================ # Pandas Converters # ============================================================================ -def to_pandas_dtypes(schema: TypeSpec) -> Dict[str, str]: +def to_pandas_dtypes(schema: TypeSpec) -> Dict[str, Any]: """ Convert TypeSpec to pandas dtypes dict. + Non-categorical fields map to pandas dtype strings; a field with + ``categories`` set maps to a real ``pd.CategoricalDtype`` instance + (item 54, gap 3) — accepted directly by ``df.astype(...)``. + Args: schema: TypeSpec to convert Returns: - Dict mapping column names to pandas dtype strings + Dict mapping column names to pandas dtype strings (or + pd.CategoricalDtype instances for categorical fields) Example: >>> schema = TypeSpec.from_simple_dict({"id": "integer", "name": "string"}) @@ -101,7 +164,18 @@ def to_pandas_dtypes(schema: TypeSpec) -> Dict[str, str]: >>> pandas_dtypes {'id': 'Int64', 'name': 'string'} """ - return {f.name: _resolve_field_native(f, TypeTarget.PANDAS) for f in schema.fields} + from mountainash.typespec._categorical import categorical_values + result: Dict[str, Any] = {} + for f in schema.fields: + if f.categories is not None: + values = categorical_values(f.categories) + import pandas as pd + result[f.name] = pd.CategoricalDtype( + categories=values, ordered=bool(f.categories_ordered) + ) + else: + result[f.name] = _resolve_field_native(f, TypeTarget.PANDAS) + return result # ============================================================================ diff --git a/tests/core/dtypes/test_target_narwhals.py b/tests/core/dtypes/test_target_narwhals.py index ec16d00b..7a8d10c9 100644 --- a/tests/core/dtypes/test_target_narwhals.py +++ b/tests/core/dtypes/test_target_narwhals.py @@ -24,6 +24,16 @@ def test_bare_name_guard_is_isinstance_type(self): bare-name parse proves the guard was adapted, not copy-pasted.""" assert target_narwhals.parse_type_string("Int64") is nw.Int64 + def test_namespace_classes_present_at_pinned_floor(self): + """Floor verification (spec §2.3 / plan Task 13, 2026-08-17): the four + namespace classes the safe-eval parser constructs — Array, Enum, + Categorical, Decimal — plus the DTypeClass instantiation mechanism + were verified present (class defs + top-level exports) in the narwhals + 2.20.0 wheel (the pinned floor in pyproject.toml). No floor change was + needed; this documents the check so a future floor raise re-runs it.""" + for name in ("Array", "Enum", "Categorical", "Decimal"): + assert hasattr(nw, name), f"nw.{name} missing at resolved version" + class TestParameterizedRoundTrip: @pytest.mark.parametrize("s,expected", [ diff --git a/tests/core/dtypes/test_target_pyarrow.py b/tests/core/dtypes/test_target_pyarrow.py index 8f9f6b46..f5f3b90a 100644 --- a/tests/core/dtypes/test_target_pyarrow.py +++ b/tests/core/dtypes/test_target_pyarrow.py @@ -31,6 +31,23 @@ def test_bare_name_via_alias_unchanged(self): assert target_pyarrow.parse_type_string("string") == pa.string() +class TestSemanticallyInvalidReturnsNone: + """Syntactically-valid-but-semantically-invalid strings match their regex + but must return None (never leak a raw Arrow constructor ValueError) so + the resolver raises the typed InvalidBackendTypeError instead.""" + + @pytest.mark.parametrize("s", [ + "timestamp[badunit]", + "decimal128(999, 10)", + "decimal256(999, 10)", + "time64[badunit]", + "time32[badunit]", + "duration[badunit]", + ]) + def test_semantically_invalid_returns_none(self, s): + assert target_pyarrow.parse_type_string(s) is None + + class TestOutOfScopeStaysNone: @pytest.mark.parametrize("s", [ "list", # recursive list grammar — out of scope diff --git a/tests/relations/dag/test_resolver_dtype_fidelity_blast_radius.py b/tests/relations/dag/test_resolver_dtype_fidelity_blast_radius.py new file mode 100644 index 00000000..337e7a38 --- /dev/null +++ b/tests/relations/dag/test_resolver_dtype_fidelity_blast_radius.py @@ -0,0 +1,162 @@ +"""Item 54 blast-radius regression: item 42's empty_frame and item 53's +inline-read cast both consume the shared resolver — the whole point of the +shared-resolver design is that one fix upgrades both at once. These tests +verify that claim end-to-end, plus the cross-backend materialization parity +(spec §7 tests 7-8) and the no-raise partner regression (§5: backend_type +None/"" must still fall through after the raise policy landed). + +GREEN expectation: no new production code is required by these tests — if any +fail, that signals one of item 54's earlier tasks missed a call-site, not a +new feature to build. +""" +from __future__ import annotations + +import polars as pl +import pytest + +from mountainash.typespec.spec import FieldSpec, TypeSpec +from mountainash.typespec.universal_types import UniversalType + +# The three dtype families item 54 upgrades: parameterized backend_type +# (gap 1), categorical (gap 3), nested list (gap 2). +UPGRADED_SPEC = TypeSpec(fields=[ + FieldSpec( + name="ts", type=UniversalType.ANY, + backend_type="Datetime(time_unit='us', time_zone='UTC')", + ), + FieldSpec( + name="cat", type=UniversalType.STRING, + categories=["a", "b"], categories_ordered=True, + ), + FieldSpec( + name="lst", type=UniversalType.ARRAY, item_type="integer", + ), +]) + +# Same shape as a Frictionless descriptor (inline-read schema dict). +UPGRADED_SCHEMA_DICT = {"fields": [ + {"name": "ts", "type": "any", + "x-mountainash": {"backend_type": "Datetime(time_unit='us', time_zone='UTC')"}}, + {"name": "cat", "type": "string", "categories": ["a", "b"], "categoriesOrdered": True}, + {"name": "lst", "type": "array", "itemType": "integer"}, +]} + +ALL_NULL_DATA = {"ts": [None], "cat": [None], "lst": [None]} + + +def _polars_ext(): + from mountainash.relations.backends.relation_systems.polars.extensions_mountainash.relsys_pl_ext_ma_util import ( + MountainashPolarsExtensionRelationSystem, + ) + return MountainashPolarsExtensionRelationSystem() + + +def _narwhals_ext(): + from mountainash.relations.backends.relation_systems.narwhals.extensions_mountainash.relsys_nw_ext_ma_util import ( + MountainashNarwhalsExtensionRelationSystem, + ) + return MountainashNarwhalsExtensionRelationSystem() + + +def _ibis_ext(): + from mountainash.relations.backends.relation_systems.ibis.extensions_mountainash.relsys_ib_ext_ma_util import ( + MountainashIbisExtensionRelationSystem, + ) + return MountainashIbisExtensionRelationSystem() + + +class TestEmptyFrameUpgradedDtypes: + """item 42's empty_frame must produce the upgraded dtypes directly.""" + + def test_polars(self): + df = _polars_ext().empty_frame(UPGRADED_SPEC).collect() + assert df.schema["ts"] == pl.Datetime(time_unit="us", time_zone="UTC") + assert df.schema["cat"] == pl.Enum(["a", "b"]) + assert df.schema["lst"] == pl.List(pl.Int64) + + def test_narwhals_schema_and_executed_op(self): + import narwhals as nw + lazy = _narwhals_ext().empty_frame(UPGRADED_SPEC) + frame = lazy.collect() # executed op + assert frame.schema["ts"] == nw.Datetime(time_unit="us", time_zone="UTC") + assert frame.schema["cat"] == nw.Enum(["a", "b"]) + assert frame.schema["lst"] == nw.List(nw.Int64) + assert frame.shape == (0, 3) + + def test_ibis_schema_and_executed_op(self): + t = _ibis_ext().empty_frame(UPGRADED_SPEC) + schema = t.schema() # backend-native schema post-wrap + assert schema["ts"].is_timestamp() and schema["ts"].timezone == "UTC" + assert schema["lst"].is_array() and schema["lst"].value_type.is_int64() + # categorical is a boundary conversion through Arrow (dictionary or + # string) — assert it is one of the honest representations, not wrong + # data; the executed op below is the parity proof. + assert schema["cat"].is_string() or schema["cat"].is_dictionary() + out = t.execute() # executed op + assert out.shape == (0, 3) + assert list(out.columns) == ["ts", "cat", "lst"] + + +class TestInlineReadUpgradedDtypes: + """item 53's inline-read cast path must produce the upgraded dtypes, + cross-backend (review Minor 2: closes the Polars-only asymmetry).""" + + def _ext(self, backend): + factories = { + "polars": _polars_ext, + "narwhals": _narwhals_ext, + "ibis": _ibis_ext, + } + return factories[backend]() + + def _collect_pl(self, native) -> pl.DataFrame: + if hasattr(native, "to_pyarrow") and not hasattr(native, "collect"): # ibis + return pl.from_arrow(native.to_pyarrow()) + if hasattr(native, "collect"): + native = native.collect() + if hasattr(native, "to_native"): # narwhals -> pl.DataFrame + native = native.to_native() + assert isinstance(native, pl.DataFrame) + return native + + @pytest.mark.parametrize("backend", ["polars", "narwhals", "ibis"]) + def test_all_null_inline_data(self, backend): + from mountainash.typespec.datapackage import DataResource + res = DataResource( + name="t", format="json", data=ALL_NULL_DATA, schema=UPGRADED_SCHEMA_DICT, + ) + df = self._collect_pl(self._ext(backend).read_resource(res)) + assert df.schema["ts"] == pl.Datetime(time_unit="us", time_zone="UTC") + assert df.schema["lst"] == pl.List(pl.Int64) + # Categorical is a genuine type-boundary conversion through the + # ibis/Arrow memtable (backlog note: Enum becomes a dictionary/string + # array — not guaranteed dtype preservation); polars/narwhals keep the + # Enum because narwhals wraps the cast polars frame with no Arrow hop. + if backend == "ibis": + assert df.schema["cat"] == pl.String + else: + assert df.schema["cat"] == pl.Enum(["a", "b"]) + + +class TestNoRaisePartnerRegression: + """§5's previously-legitimate half: backend_type None/"" on a non-ANY + canonical type must still fall through to canonical — both consumers + complete without raising InvalidBackendTypeError.""" + + @pytest.mark.parametrize("backend_type", [None, ""]) + def test_empty_frame_no_raise(self, backend_type): + spec = TypeSpec(fields=[ + FieldSpec(name="i", type=UniversalType.INTEGER, backend_type=backend_type), + ]) + df = _polars_ext().empty_frame(spec).collect() + assert df.schema["i"] == pl.Int64 + + @pytest.mark.parametrize("backend_type", [None, ""]) + def test_inline_read_no_raise(self, backend_type): + from mountainash.typespec.datapackage import DataResource + schema = {"fields": [ + {"name": "i", "type": "integer", "x-mountainash": {"backend_type": backend_type}}, + ]} + res = DataResource(name="t", format="json", data={"i": [None]}, schema=schema) + df = _polars_ext().read_resource(res).collect() + assert df.schema["i"] == pl.Int64 diff --git a/tests/typespec/test_categorical.py b/tests/typespec/test_categorical.py new file mode 100644 index 00000000..372f5f01 --- /dev/null +++ b/tests/typespec/test_categorical.py @@ -0,0 +1,59 @@ +"""Tests for the shared categorical-values extraction helper. + +Two kinds of tests live here: +- Genuinely RED-first unit tests for ``categorical_values`` (the helper did + not exist before item 54 — these drive it into existence). +- A characterization (golden-master) test proving the conform call-site swap + (item 54, task 9) is behavior-preserving: it records the output conform's + OLD inline extraction produced for identical fixtures and asserts the shared + helper matches, so a refactor regression shows up as a real failure rather + than being asserted-away. +""" +from __future__ import annotations + +from mountainash.typespec._categorical import categorical_values + +# Baseline recorded from conform/expressions.py:958-964's inline extraction +# (pre-refactor, 2026-08-17): simple -> ['a', 'b']; object -> [0, 1]; +# mixed -> [0, 'x']. Identical fixtures, asserted equal below. +_BASELINE = { + "simple": ["a", "b"], + "object": [{"value": 0, "label": "Low"}, {"value": 1, "label": "High"}], + "mixed": [{"value": 0, "label": "Low"}, "x"], +} +_BASELINE_OUTPUT = { + "simple": ["a", "b"], + "object": [0, 1], + "mixed": [0, "x"], +} + + +class TestCategoricalValues: + def test_simple_form(self): + assert categorical_values(["a", "b"]) == ["a", "b"] + + def test_object_form(self): + assert categorical_values( + [{"value": 0, "label": "Low"}, {"value": 1, "label": "High"}] + ) == [0, 1] + + def test_mixed_form(self): + assert categorical_values([{"value": 0, "label": "Low"}, "x"]) == [0, "x"] + + def test_empty(self): + assert categorical_values([]) == [] + + def test_returns_new_list(self): + cats = ["a", "b"] + result = categorical_values(cats) + assert result == ["a", "b"] + assert result is not cats # never aliases the input + + +class TestMatchesConformStage5bBaseline: + def test_categorical_values_matches_conform_stage_5b_extraction(self): + """Characterization, NOT new-behavior assertion: the shared helper's + output must equal what conform's old inline extraction produced for + the identical fixtures (behavior-preserving refactor proof).""" + for name, cats in _BASELINE.items(): + assert categorical_values(cats) == _BASELINE_OUTPUT[name], name diff --git a/tests/typespec/test_converters.py b/tests/typespec/test_converters.py index b422c359..ac0d914b 100644 --- a/tests/typespec/test_converters.py +++ b/tests/typespec/test_converters.py @@ -92,10 +92,43 @@ def test_all_universal_types_produce_a_result(self): assert len(result) == len(list(UniversalType)) def test_returns_dict_of_strings(self, basic_schema): + """Non-categorical fields are plain strings; a categorical field is + the deliberate exception (a real pd.CategoricalDtype instance).""" + import pandas as pd + categorical = TypeSpec(fields=[ + FieldSpec(name="cat", type=UniversalType.STRING, + categories=["a", "b"], categories_ordered=True), + ]) result = to_pandas_dtypes(basic_schema) assert isinstance(result, dict) for v in result.values(): assert isinstance(v, str) + cat_result = to_pandas_dtypes(categorical) + assert isinstance(cat_result["cat"], pd.CategoricalDtype) + assert not isinstance(cat_result["cat"], str) + + def test_categorical_field_returns_categorical_dtype(self): + """§4.3: to_pandas_dtypes returns a real pd.CategoricalDtype instance + for a categorical field — pandas accepts it directly as astype input.""" + import pandas as pd + spec = TypeSpec(fields=[ + FieldSpec(name="col", type=UniversalType.STRING, + categories=["a", "b"], categories_ordered=True), + ]) + result = to_pandas_dtypes(spec) + assert isinstance(result["col"], pd.CategoricalDtype) + assert list(result["col"].categories) == ["a", "b"] + assert result["col"].ordered is True + + def test_unordered_categorical_field_ordered_false(self): + import pandas as pd + spec = TypeSpec(fields=[ + FieldSpec(name="col", type=UniversalType.STRING, + categories=["a", "b"], categories_ordered=False), + ]) + result = to_pandas_dtypes(spec) + assert isinstance(result["col"], pd.CategoricalDtype) + assert result["col"].ordered is False # ============================================================================ @@ -205,6 +238,138 @@ def test_backend_type_preserved_in_ibis(self): assert result["val"] == "float32" +# ============================================================================ +# TestCategoricalSchema (item 54, gap 3) +# ============================================================================ + +class TestCategoricalSchema: + """Gap 3: categories/categoriesOrdered -> real Polars categorical. + + categories takes priority over backend_type/type entirely — mirrors + conform stage 5's mutually-exclusive branch ordering exactly.""" + + def _spec(self, categories, ordered=None, backend_type=None): + return TypeSpec(fields=[ + FieldSpec( + name="cat", + type=UniversalType.STRING, + categories=categories, + categories_ordered=ordered, + backend_type=backend_type, + ), + ]) + + def test_unordered_categories_is_pl_categorical(self): + import polars as pl + result = to_polars_schema(self._spec(["a", "b"], ordered=False)) + assert result["cat"] is pl.Categorical + + def test_ordered_categories_is_pl_enum(self): + import polars as pl + result = to_polars_schema(self._spec(["a", "b"], ordered=True)) + assert result["cat"] == pl.Enum(["a", "b"]) + + def test_object_form_categories_use_shared_extraction(self): + """Object-form categories must extract identically to conform's + stage-5b (shared categorical_values helper — no drift).""" + import polars as pl + from mountainash.typespec._categorical import categorical_values + cats = [{"value": 0, "label": "Low"}, {"value": 1, "label": "High"}] + result = to_polars_schema(self._spec(cats, ordered=True)) + assert result["cat"] == pl.Enum([str(v) for v in categorical_values(cats)]) + + def test_categories_win_over_invalid_backend_type(self): + """Precedence (spec §5): a field with BOTH categories set AND an + invalid backend_type takes the categorical branch and never raises — + the backend_type is never even parsed for such a field.""" + import polars as pl + result = to_polars_schema(self._spec(["a"], ordered=False, backend_type="garbage")) + assert result["cat"] is pl.Categorical + + def test_ibis_categories_stay_string(self): + """Ibis has no categorical primitive — categories present still + resolves to string (explicit, not silently untested).""" + result = to_ibis_schema(self._spec(["a", "b"], ordered=True)) + assert result["cat"] == "string" + + +# ============================================================================ +# TestNestedListItemType (item 54, gap 2) +# ============================================================================ + +class TestNestedListItemType: + """Gap 2: nested LIST inner type via the existing FieldSpec.item_type. + + item_type is a Frictionless-standard carriage (spec §list) already carried + on FieldSpec — the resolver previously never read it, so ARRAY resolved to + a bare container (and PyArrow silently defaulted every untyped list to a + string element).""" + + def _spec(self, item_type=None): + return TypeSpec(fields=[ + FieldSpec(name="lst", type=UniversalType.ARRAY, item_type=item_type), + ]) + + def test_polars_item_type_resolves_inner(self): + import polars as pl + result = to_polars_schema(self._spec("integer")) + assert result["lst"] == pl.List(pl.Int64) + + def test_narwhals_item_type_resolves_inner(self): + import narwhals as nw + result = to_polars_schema(self._spec("integer")) + # narwhals wraps the already-cast polars-native frame on the live + # consumers (empty_frame / inline-read); assert the narwhals-native + # form of the same inner type is reachable via the registry. + from mountainash.core.dtypes import TypeTarget, registry + from mountainash.typespec.converters import _resolve_field_native + native = _resolve_field_native(self._spec("integer").fields[0], TypeTarget.NARWHALS) + assert native == nw.List(nw.Int64) + assert native is not nw.List # real parameterized instance, not bare class + + def test_pyarrow_item_type_resolves_inner_not_string(self): + """Second latent bug regression: the bare fallback silently defaulted + every untyped list to a string element. With item_type the inner must + be the real element type.""" + pytest.importorskip("pyarrow") + import pyarrow as pa + result = to_arrow_schema(self._spec("integer")) + field = result.field("lst") + assert field.type == pa.list_(pa.int64()) + assert field.type.value_type == pa.int64() # NOT pa.string() + + def test_ibis_item_type_resolves_inner(self): + result = to_ibis_schema(self._spec("integer")) + assert result["lst"] == "array" + + def test_pandas_stays_object(self): + """Pandas has no native parameterized list dtype — 'object' is the + correct, only representation (regression lock, not a gap).""" + result = to_pandas_dtypes(self._spec("integer")) + assert result["lst"] == "object" + + @pytest.mark.parametrize("item_type", [None, "any"]) + def test_no_parameterization_keeps_bare_container(self, item_type): + """No item_type (or item_type='any' — same code path) -> bare + container, unchanged (regression).""" + import polars as pl + result = to_polars_schema(self._spec(item_type)) + assert result["lst"] is pl.List + + def test_unknown_item_type_raises_with_field_context_and_chain(self): + """item_type='garbage' is a second raise surface beyond + InvalidBackendTypeError: UnknownDtypeError naming the field, chained + to the original parse_universal error (chain must not be dropped).""" + from mountainash.core.dtypes import UnknownDtypeError + with pytest.raises(UnknownDtypeError) as exc_info: + to_polars_schema(self._spec("garbage")) + assert "lst" in str(exc_info.value) + assert "garbage" in str(exc_info.value) + # chain: __cause__ is the original UnknownDtypeError parse_universal + # raised, not swallowed by a message-only copy + assert isinstance(exc_info.value.__cause__, UnknownDtypeError) + + # ============================================================================ # TestConvertersOverRegistry # ============================================================================ @@ -266,6 +431,22 @@ def test_empty_or_none_backend_type_falls_through(self, backend_type): ]) assert to_polars_schema(spec)["x"] is pl.Int64 + def test_semantically_invalid_backend_type_raises_typed_error_not_raw(self): + """Regression (GLM-5.2 whole-branch review, Important finding 1): a + syntactically-valid-but-semantically-invalid PyArrow string + (timestamp[badunit]) must surface as the typed InvalidBackendTypeError + through the converter, never a raw Arrow constructor ValueError.""" + from mountainash.core.dtypes import InvalidBackendTypeError + from mountainash.typespec import TypeSpec, FieldSpec + from mountainash.typespec.universal_types import UniversalType + from mountainash.typespec.converters import to_arrow_schema + spec = TypeSpec(fields=[ + FieldSpec(name="x", type=UniversalType.INTEGER, + backend_type="timestamp[badunit]"), + ]) + with pytest.raises(InvalidBackendTypeError, match="timestamp\[badunit\]"): + to_arrow_schema(spec) + def test_backend_type_preferred_when_parseable(self): import polars as pl from mountainash.typespec import TypeSpec, FieldSpec