feat(typespec): nested OBJECT/STRUCT schema fidelity (item 102)

docs/reference/expression-coverage.json

@@ -70233,6 +70233,19 @@
       "upstream_ref": null,
       "since": "2026-08-06"
     },
+    {
+      "id": "MA-CONF-04",
+      "kind": "engine_leniency",
+      "operation_keys": [],
+      "backends": [
+        "ibis-sqlite"
+      ],
+      "summary": "ibis-sqlite cannot construct struct-typed tables at all — UnsupportedBackendType('Struct types aren't supported in SQLite') on table creation, struct literals, and string-to-struct casts",
+      "impact": "any test constructing a struct-typed source table on ibis-sqlite raises UnsupportedBackendType; ibis-duckdb/ibis-polars and polars/narwhals construct it (resolver schema-string construction for to_ibis_schema stays exercised — only real-table execution is gated)",
+      "workaround": "Use ibis-duckdb/ibis-polars or a polars/narwhals backend for struct-typed data",
+      "upstream_ref": null,
+      "since": "2026-08-18"
+    },
     {
       "id": "MA-MATH-01",
       "kind": "precision",

docs/reference/expression-coverage.md

@@ -551,6 +551,7 @@ Cells whose facts are all scoped (dialect / parameter / option / value-class) ha
 | MA-CONF-01 | engine_leniency | pandas, narwhals-pandas, ibis-sqlite | — | conform struct dotted-source extraction is unsupported on pandas/narwhals-pandas (TypeError) and ibis-sqlite (UnsupportedBackendType) — no native struct column type | conform() with a dotted struct source path raises on pandas/narwhals-pandas/ibis-sqlite; polars and ibis-duckdb/ibis-polars extract it | Use a polars backend or ibis-duckdb/ibis-polars for struct dotted-source conform | — | 2026-08-06 |
 | MA-CONF-02 | engine_leniency | pandas, narwhals, ibis-sqlite | — | conform discard-value/discard-row drift policies are unsupported on pandas/narwhals (unenriched BackendCapabilityError) and ibis-sqlite (OperationNotDefinedError) | conform() discard_value/discard_row policies raise on pandas/narwhals and ibis-sqlite; polars and ibis-duckdb/ibis-polars apply them | Use a polars backend or ibis-duckdb/ibis-polars for discard drift policies | — | 2026-08-06 |
 | MA-CONF-03 | engine_leniency | ibis | — | conform multi-transform full pipeline raises IbisTypeError on all ibis backends (deferred type resolution rejects the chained transform) | a full conform multi-transform pipeline raises on ibis-duckdb/ibis-polars/ibis-sqlite; polars/narwhals run it | Use a polars or narwhals backend for full conform transform pipelines | — | 2026-08-06 |
+| MA-CONF-04 | engine_leniency | ibis-sqlite | — | ibis-sqlite cannot construct struct-typed tables at all — UnsupportedBackendType('Struct types aren't supported in SQLite') on table creation, struct literals, and string-to-struct casts | any test constructing a struct-typed source table on ibis-sqlite raises UnsupportedBackendType; ibis-duckdb/ibis-polars and polars/narwhals construct it (resolver schema-string construction for to_ibis_schema stays exercised — only real-table execution is gated) | Use ibis-duckdb/ibis-polars or a polars/narwhals backend for struct-typed data | — | 2026-08-18 |
 | MA-MATH-01 | precision | polars, narwhals, ibis | — | Intermediate float precision and rounding differ across backends | Exact equality comparisons on float results can fail across backends | Use is_close(precision=...) instead of eq() for float comparisons | MA-MATH-01 | 2026-07-05 |
 | MA-MATH-02 | semantics | polars, narwhals, ibis, pandas | — | cbrt() of a negative value returns NaN on every backend (the pow(x, 1/3) implementation is undefined for negatives), instead of the real cube root | ma.col(x).cbrt() on negative inputs yields NaN across all backends; a mathematically-correct negative cube root is not available | Compute sign(x) * abs(x) ** (1/3) manually for negative inputs | — | 2026-08-06 |
 | MA-REL-01 | engine_leniency | ibis, narwhals-lazy | — | pivot (long-to-wide) is unsupported on ibis (TypeError) and narwhals-lazy (AttributeError) | Relation.pivot() raises on all ibis backends and narwhals-lazy; polars and eager narwhals compute it | Use a polars or eager narwhals backend for pivot | — | 2026-08-06 |

src/mountainash/conform/expressions.py

@@ -959,10 +959,11 @@ def _build_field_expr(
         from mountainash.typespec._categorical import categorical_values
         cat_values = categorical_values(fld.categories)
 
-        # Step 1: base type cast (if needed)
-        if fld.type and fld.type != UniversalType.ANY:
+        # OBJECT has no scalar base cast; categories still take precedence
+        # over object_fields for this degenerate declaration.
+        if fld.type and fld.type not in (UniversalType.ANY, UniversalType.OBJECT):
             canon = to_canonical(fld.type)
-            if canon is not None:  # ANY -> no cast (guard already excludes ANY)
+            if canon is not None:
                 expr = expr.cast(canon)
 
         # Step 2: categorical wrapper (Polars-specific)
@@ -1005,6 +1006,18 @@ def _build_field_expr(
             .when(str_expr.is_in(*false_vals)).then(ma.lit(False))
             .otherwise(ma.lit(None))
         )
+    # Stage 5e: STRUCT — cast an already-struct-typed source column to the
+    # fully nested typed struct (item 102). Source is assumed to already be
+    # native struct/dict-shaped; this is not a JSON-string parse path.
+    elif fld.type == UniversalType.OBJECT and fld.object_fields:
+        from mountainash.core.dtypes import TypeTarget
+        from mountainash.typespec.converters import _resolve_struct_inner
+
+        native_struct = _resolve_struct_inner(
+            fld.name, fld.object_fields, TypeTarget.POLARS, None
+        )
+        expr = expr.cast(native_struct)
+
 
     # Stage 5d: DEFAULT TYPE CAST
     # Branches on type_action (item 48 Task 6 data_type policy): "coerce"

src/mountainash/core/capabilities/divergences.py

@@ -735,6 +735,17 @@ def _all() -> tuple[DivergenceFact, ...]:
             upstream_ref=None,
             since="2026-08-06",
         ),
+        DivergenceFact(
+            id="MA-CONF-04",
+            kind=DivergenceKind.ENGINE_LENIENCY,
+            operation_keys=(),  # typespec/conform struct materialization
+            backends=("ibis-sqlite",),
+            summary="ibis-sqlite cannot construct struct-typed tables at all — UnsupportedBackendType('Struct types aren't supported in SQLite') on table creation, struct literals, and string-to-struct casts",
+            impact="any test constructing a struct-typed source table on ibis-sqlite raises UnsupportedBackendType; ibis-duckdb/ibis-polars and polars/narwhals construct it (resolver schema-string construction for to_ibis_schema stays exercised — only real-table execution is gated)",
+            workaround="Use ibis-duckdb/ibis-polars or a polars/narwhals backend for struct-typed data",
+            upstream_ref=None,
+            since="2026-08-18",
+        ),
         DivergenceFact(
             id="MA-TERN-01",
             kind=DivergenceKind.ENGINE_LENIENCY,

src/mountainash/typespec/converters.py

@@ -32,30 +32,57 @@
 # ============================================================================
 
 def _resolve_field_native(field: "FieldSpec", target: TypeTarget) -> Any:
-    """backend_type (if the target can parse it) > FieldSpec.type via canon.
-
-    ANY materializes as STRING (documented default — a target schema must be
-    complete). No silent warn-and-default fallbacks.
-    """
+    """Resolve categories, backend overrides, canonical types, and containers."""
+    if field.categories is not None and target in (TypeTarget.POLARS, TypeTarget.PANDAS):
+        from mountainash.typespec._categorical import categorical_values
+        values = categorical_values(field.categories)
+        if target is TypeTarget.POLARS:
+            from mountainash.core.lazy_imports import import_polars
+            pl = import_polars()
+            return pl.Enum([str(v) for v in values]) if field.categories_ordered else pl.Categorical
+        import pandas as pd
+        return pd.CategoricalDtype(categories=values, ordered=bool(field.categories_ordered))
     if field.backend_type:
         parsed = registry.parse_type_string(field.backend_type, target)
         if parsed is not None:
             return parsed
-        # Validation strictness (item 54, §5): only a non-empty, non-None
-        # backend_type that the target cannot parse raises. None/"" means "no
-        # override given" and falls through to canonical (item 53's ANY->STRING
-        # case relies on that).
         raise InvalidBackendTypeError(field.name, field.backend_type, target)
     canon = to_canonical(field.type)
-    if canon is None:  # ANY
+    if canon is None:
         canon = MountainashDtype.STRING
     native = registry.to_native_schema(canon, target)
-
     if canon is MountainashDtype.LIST and field.item_type:
         native = _resolve_list_inner(field.name, field.item_type, target, native)
+    elif canon is MountainashDtype.STRUCT and field.object_fields:
+        native = _resolve_struct_inner(field.name, field.object_fields, target, native)
     return native
 
 
+def _resolve_struct_inner(
+    field_name: str, object_fields: list["FieldSpec"], target: TypeTarget, bare_native: Any,
+) -> Any:
+    """Build a fully parameterized native struct dtype from nested FieldSpecs."""
+    if target is TypeTarget.PANDAS:
+        return bare_native
+    inner_pairs = [(f.name, _resolve_field_native(f, target)) for f in object_fields]
+    if target is TypeTarget.POLARS:
+        from mountainash.core.lazy_imports import import_polars
+        pl = import_polars()
+        return pl.Struct({name: native for name, native in inner_pairs})
+    if target is TypeTarget.NARWHALS:
+        from mountainash.core.lazy_imports import import_narwhals
+        nw = import_narwhals()
+        return nw.Struct({name: native for name, native in inner_pairs})
+    if target is TypeTarget.PYARROW:
+        from mountainash.core.lazy_imports import import_pyarrow
+        pa = import_pyarrow()
+        return pa.struct([pa.field(name, native) for name, native in inner_pairs])
+    if target is TypeTarget.IBIS:
+        inner_str = ", ".join(f"{name}: {native}" for name, native in inner_pairs)
+        return f"struct<{inner_str}>"
+    return bare_native
+
+
 def _resolve_list_inner(
     field_name: str, item_type_str: str, target: TypeTarget, bare_native: Any
 ) -> Any:
@@ -120,22 +147,12 @@ def to_polars_schema(schema: TypeSpec) -> Dict[str, Any]:
         {'id': Int64, 'name': Utf8}
     """
     from mountainash.core.lazy_imports import import_polars
-    from mountainash.typespec._categorical import categorical_values
     pl = import_polars()
     if pl is None:
         raise ImportError("polars is required for to_polars_schema()")
     result = {}
     for f in schema.fields:
-        if f.categories is not None:
-            # categories takes priority over backend_type/type entirely
-            # (mirrors conform stage 5's branch order exactly).
-            values = categorical_values(f.categories)
-            result[f.name] = (
-                pl.Enum([str(v) for v in values]) if f.categories_ordered
-                else pl.Categorical
-            )
-        else:
-            result[f.name] = _resolve_field_native(f, TypeTarget.POLARS)
+        result[f.name] = _resolve_field_native(f, TypeTarget.POLARS)
     return result
 
 
@@ -164,17 +181,9 @@ def to_pandas_dtypes(schema: TypeSpec) -> Dict[str, Any]:
         >>> pandas_dtypes
         {'id': 'Int64', 'name': 'string'}
     """
-    from mountainash.typespec._categorical import categorical_values
     result: Dict[str, Any] = {}
     for f in schema.fields:
-        if f.categories is not None:
-            values = categorical_values(f.categories)
-            import pandas as pd
-            result[f.name] = pd.CategoricalDtype(
-                categories=values, ordered=bool(f.categories_ordered)
-            )
-        else:
-            result[f.name] = _resolve_field_native(f, TypeTarget.PANDAS)
+        result[f.name] = _resolve_field_native(f, TypeTarget.PANDAS)
     return result
 
 

src/mountainash/typespec/extraction.py

@@ -430,6 +430,27 @@ def extract_from_dataframe(
 from_dataframe = extract_from_dataframe
 
 
+def _fields_from_polars_struct(dtype: "pl.Struct") -> list["FieldSpec"]:
+    """Recursively build FieldSpec.object_fields from a Polars struct."""
+    from mountainash.core.lazy_imports import import_polars
+    pl = import_polars()
+    fields = []
+    for f in dtype.fields:
+        name, inner = f.name, f.dtype
+        item_type = None
+        if isinstance(inner, pl.List) and inner.inner is not None:
+            inner_universal, _ = from_canonical(
+                registry.from_native(inner.inner, target=TypeTarget.POLARS)
+            )
+            item_type = inner_universal.value
+        object_fields = _fields_from_polars_struct(inner) if isinstance(inner, pl.Struct) else None
+        universal_type, _ = from_canonical(registry.from_native(inner, target=TypeTarget.POLARS))
+        fields.append(FieldSpec(
+            name=name, type=universal_type, item_type=item_type, object_fields=object_fields,
+        ))
+    return fields
+
+
 def _from_polars(df: 'pl.DataFrame', preserve_backend_types: bool, **metadata) -> TypeSpec:
     """Extract schema from Polars DataFrame or LazyFrame."""
     from mountainash.core.lazy_imports import import_polars
@@ -443,23 +464,24 @@ def _from_polars(df: 'pl.DataFrame', preserve_backend_types: bool, **metadata) -
     schema_dict = df.schema
 
     for col_name, dtype in schema_dict.items():
-        # Get backend type name
-        backend_type_str = str(dtype)
-
-        # Convert to universal type
         universal_type = _universal_from_native(dtype, TypeTarget.POLARS)
 
         item_type = None
+        object_fields = None
         if isinstance(dtype, pl.List) and dtype.inner is not None:
             inner_universal, _ = from_canonical(
                 registry.from_native(dtype.inner, target=TypeTarget.POLARS)
             )
             item_type = inner_universal.value
+        elif isinstance(dtype, pl.Struct):
+            object_fields = _fields_from_polars_struct(dtype)
 
+        backend_type_str = None if isinstance(dtype, pl.Struct) else str(dtype)
         schema_field = FieldSpec(
             name=col_name,
             type=universal_type,
             item_type=item_type,
+            object_fields=object_fields,
             backend_type=backend_type_str if preserve_backend_types else None,
         )
         fields.append(schema_field)
@@ -505,6 +527,27 @@ def _from_pandas(df: 'pd.DataFrame', preserve_backend_types: bool, **metadata) -
     )
 
 
+def _fields_from_pyarrow_struct(dtype: "pa.StructType") -> list["FieldSpec"]:
+    """Recursively build FieldSpec.object_fields from a PyArrow struct."""
+    from mountainash.core.lazy_imports import import_pyarrow
+    pa = import_pyarrow()
+    fields = []
+    for f in dtype:
+        name, inner = f.name, f.type
+        item_type = None
+        if pa.types.is_list(inner):
+            inner_universal, _ = from_canonical(
+                registry.from_native(inner.value_type, target=TypeTarget.PYARROW)
+            )
+            item_type = inner_universal.value
+        object_fields = _fields_from_pyarrow_struct(inner) if pa.types.is_struct(inner) else None
+        universal_type, _ = from_canonical(registry.from_native(inner, target=TypeTarget.PYARROW))
+        fields.append(FieldSpec(
+            name=name, type=universal_type, item_type=item_type, object_fields=object_fields,
+        ))
+    return fields
+
+
 def _from_pyarrow(table: 'pa.Table', preserve_backend_types: bool, **metadata) -> TypeSpec:
     """Extract schema from PyArrow Table."""
     from mountainash.core.lazy_imports import import_pyarrow
@@ -515,26 +558,25 @@ def _from_pyarrow(table: 'pa.Table', preserve_backend_types: bool, **metadata) -
     fields = []
 
     for field in table.schema:
-        # Get backend type
         backend_type = field.type
-
-        # Convert to string for normalization
-        backend_type_str = str(backend_type)
-
-        # Convert to universal type
         universal_type = _universal_from_native(backend_type, TypeTarget.PYARROW)
 
         item_type = None
+        object_fields = None
         if pa.types.is_list(backend_type):
             inner_universal, _ = from_canonical(
                 registry.from_native(backend_type.value_type, target=TypeTarget.PYARROW)
             )
             item_type = inner_universal.value
+        elif pa.types.is_struct(backend_type):
+            object_fields = _fields_from_pyarrow_struct(backend_type)
 
+        backend_type_str = None if pa.types.is_struct(backend_type) else str(backend_type)
         schema_field = FieldSpec(
             name=field.name,
             type=universal_type,
             item_type=item_type,
+            object_fields=object_fields,
             backend_type=backend_type_str if preserve_backend_types else None,
         )
         fields.append(schema_field)