diff --git a/docs/reference/expression-coverage.json b/docs/reference/expression-coverage.json index c5fa843e..dc5cc1fa 100644 --- a/docs/reference/expression-coverage.json +++ b/docs/reference/expression-coverage.json @@ -70233,6 +70233,19 @@ "upstream_ref": null, "since": "2026-08-06" }, + { + "id": "MA-CONF-04", + "kind": "engine_leniency", + "operation_keys": [], + "backends": [ + "ibis-sqlite" + ], + "summary": "ibis-sqlite cannot construct struct-typed tables at all — UnsupportedBackendType('Struct types aren't supported in SQLite') on table creation, struct literals, and string-to-struct casts", + "impact": "any test constructing a struct-typed source table on ibis-sqlite raises UnsupportedBackendType; ibis-duckdb/ibis-polars and polars/narwhals construct it (resolver schema-string construction for to_ibis_schema stays exercised — only real-table execution is gated)", + "workaround": "Use ibis-duckdb/ibis-polars or a polars/narwhals backend for struct-typed data", + "upstream_ref": null, + "since": "2026-08-18" + }, { "id": "MA-MATH-01", "kind": "precision", diff --git a/docs/reference/expression-coverage.md b/docs/reference/expression-coverage.md index ef5428f7..078d1aa7 100644 --- a/docs/reference/expression-coverage.md +++ b/docs/reference/expression-coverage.md @@ -551,6 +551,7 @@ Cells whose facts are all scoped (dialect / parameter / option / value-class) ha | MA-CONF-01 | engine_leniency | pandas, narwhals-pandas, ibis-sqlite | — | conform struct dotted-source extraction is unsupported on pandas/narwhals-pandas (TypeError) and ibis-sqlite (UnsupportedBackendType) — no native struct column type | conform() with a dotted struct source path raises on pandas/narwhals-pandas/ibis-sqlite; polars and ibis-duckdb/ibis-polars extract it | Use a polars backend or ibis-duckdb/ibis-polars for struct dotted-source conform | — | 2026-08-06 | | MA-CONF-02 | engine_leniency | pandas, narwhals, ibis-sqlite | — | conform discard-value/discard-row drift policies are unsupported on pandas/narwhals (unenriched BackendCapabilityError) and ibis-sqlite (OperationNotDefinedError) | conform() discard_value/discard_row policies raise on pandas/narwhals and ibis-sqlite; polars and ibis-duckdb/ibis-polars apply them | Use a polars backend or ibis-duckdb/ibis-polars for discard drift policies | — | 2026-08-06 | | MA-CONF-03 | engine_leniency | ibis | — | conform multi-transform full pipeline raises IbisTypeError on all ibis backends (deferred type resolution rejects the chained transform) | a full conform multi-transform pipeline raises on ibis-duckdb/ibis-polars/ibis-sqlite; polars/narwhals run it | Use a polars or narwhals backend for full conform transform pipelines | — | 2026-08-06 | +| MA-CONF-04 | engine_leniency | ibis-sqlite | — | ibis-sqlite cannot construct struct-typed tables at all — UnsupportedBackendType('Struct types aren't supported in SQLite') on table creation, struct literals, and string-to-struct casts | any test constructing a struct-typed source table on ibis-sqlite raises UnsupportedBackendType; ibis-duckdb/ibis-polars and polars/narwhals construct it (resolver schema-string construction for to_ibis_schema stays exercised — only real-table execution is gated) | Use ibis-duckdb/ibis-polars or a polars/narwhals backend for struct-typed data | — | 2026-08-18 | | MA-MATH-01 | precision | polars, narwhals, ibis | — | Intermediate float precision and rounding differ across backends | Exact equality comparisons on float results can fail across backends | Use is_close(precision=...) instead of eq() for float comparisons | MA-MATH-01 | 2026-07-05 | | MA-MATH-02 | semantics | polars, narwhals, ibis, pandas | — | cbrt() of a negative value returns NaN on every backend (the pow(x, 1/3) implementation is undefined for negatives), instead of the real cube root | ma.col(x).cbrt() on negative inputs yields NaN across all backends; a mathematically-correct negative cube root is not available | Compute sign(x) * abs(x) ** (1/3) manually for negative inputs | — | 2026-08-06 | | MA-REL-01 | engine_leniency | ibis, narwhals-lazy | — | pivot (long-to-wide) is unsupported on ibis (TypeError) and narwhals-lazy (AttributeError) | Relation.pivot() raises on all ibis backends and narwhals-lazy; polars and eager narwhals compute it | Use a polars or eager narwhals backend for pivot | — | 2026-08-06 | diff --git a/src/mountainash/conform/expressions.py b/src/mountainash/conform/expressions.py index ca834f78..7a5df0e7 100644 --- a/src/mountainash/conform/expressions.py +++ b/src/mountainash/conform/expressions.py @@ -959,10 +959,11 @@ def _build_field_expr( from mountainash.typespec._categorical import categorical_values cat_values = categorical_values(fld.categories) - # Step 1: base type cast (if needed) - if fld.type and fld.type != UniversalType.ANY: + # OBJECT has no scalar base cast; categories still take precedence + # over object_fields for this degenerate declaration. + if fld.type and fld.type not in (UniversalType.ANY, UniversalType.OBJECT): canon = to_canonical(fld.type) - if canon is not None: # ANY -> no cast (guard already excludes ANY) + if canon is not None: expr = expr.cast(canon) # Step 2: categorical wrapper (Polars-specific) @@ -1005,6 +1006,18 @@ def _build_field_expr( .when(str_expr.is_in(*false_vals)).then(ma.lit(False)) .otherwise(ma.lit(None)) ) + # Stage 5e: STRUCT — cast an already-struct-typed source column to the + # fully nested typed struct (item 102). Source is assumed to already be + # native struct/dict-shaped; this is not a JSON-string parse path. + elif fld.type == UniversalType.OBJECT and fld.object_fields: + from mountainash.core.dtypes import TypeTarget + from mountainash.typespec.converters import _resolve_struct_inner + + native_struct = _resolve_struct_inner( + fld.name, fld.object_fields, TypeTarget.POLARS, None + ) + expr = expr.cast(native_struct) + # Stage 5d: DEFAULT TYPE CAST # Branches on type_action (item 48 Task 6 data_type policy): "coerce" diff --git a/src/mountainash/core/capabilities/divergences.py b/src/mountainash/core/capabilities/divergences.py index 8aeabcbb..b2cd4053 100644 --- a/src/mountainash/core/capabilities/divergences.py +++ b/src/mountainash/core/capabilities/divergences.py @@ -735,6 +735,17 @@ def _all() -> tuple[DivergenceFact, ...]: upstream_ref=None, since="2026-08-06", ), + DivergenceFact( + id="MA-CONF-04", + kind=DivergenceKind.ENGINE_LENIENCY, + operation_keys=(), # typespec/conform struct materialization + backends=("ibis-sqlite",), + summary="ibis-sqlite cannot construct struct-typed tables at all — UnsupportedBackendType('Struct types aren't supported in SQLite') on table creation, struct literals, and string-to-struct casts", + impact="any test constructing a struct-typed source table on ibis-sqlite raises UnsupportedBackendType; ibis-duckdb/ibis-polars and polars/narwhals construct it (resolver schema-string construction for to_ibis_schema stays exercised — only real-table execution is gated)", + workaround="Use ibis-duckdb/ibis-polars or a polars/narwhals backend for struct-typed data", + upstream_ref=None, + since="2026-08-18", + ), DivergenceFact( id="MA-TERN-01", kind=DivergenceKind.ENGINE_LENIENCY, diff --git a/src/mountainash/typespec/converters.py b/src/mountainash/typespec/converters.py index 560e6089..ddd51b17 100644 --- a/src/mountainash/typespec/converters.py +++ b/src/mountainash/typespec/converters.py @@ -32,30 +32,57 @@ # ============================================================================ def _resolve_field_native(field: "FieldSpec", target: TypeTarget) -> Any: - """backend_type (if the target can parse it) > FieldSpec.type via canon. - - ANY materializes as STRING (documented default — a target schema must be - complete). No silent warn-and-default fallbacks. - """ + """Resolve categories, backend overrides, canonical types, and containers.""" + if field.categories is not None and target in (TypeTarget.POLARS, TypeTarget.PANDAS): + from mountainash.typespec._categorical import categorical_values + values = categorical_values(field.categories) + if target is TypeTarget.POLARS: + from mountainash.core.lazy_imports import import_polars + pl = import_polars() + return pl.Enum([str(v) for v in values]) if field.categories_ordered else pl.Categorical + import pandas as pd + return pd.CategoricalDtype(categories=values, ordered=bool(field.categories_ordered)) if field.backend_type: parsed = registry.parse_type_string(field.backend_type, target) if parsed is not None: return parsed - # Validation strictness (item 54, §5): only a non-empty, non-None - # backend_type that the target cannot parse raises. None/"" means "no - # override given" and falls through to canonical (item 53's ANY->STRING - # case relies on that). raise InvalidBackendTypeError(field.name, field.backend_type, target) canon = to_canonical(field.type) - if canon is None: # ANY + if canon is None: canon = MountainashDtype.STRING native = registry.to_native_schema(canon, target) - if canon is MountainashDtype.LIST and field.item_type: native = _resolve_list_inner(field.name, field.item_type, target, native) + elif canon is MountainashDtype.STRUCT and field.object_fields: + native = _resolve_struct_inner(field.name, field.object_fields, target, native) return native +def _resolve_struct_inner( + field_name: str, object_fields: list["FieldSpec"], target: TypeTarget, bare_native: Any, +) -> Any: + """Build a fully parameterized native struct dtype from nested FieldSpecs.""" + if target is TypeTarget.PANDAS: + return bare_native + inner_pairs = [(f.name, _resolve_field_native(f, target)) for f in object_fields] + if target is TypeTarget.POLARS: + from mountainash.core.lazy_imports import import_polars + pl = import_polars() + return pl.Struct({name: native for name, native in inner_pairs}) + if target is TypeTarget.NARWHALS: + from mountainash.core.lazy_imports import import_narwhals + nw = import_narwhals() + return nw.Struct({name: native for name, native in inner_pairs}) + if target is TypeTarget.PYARROW: + from mountainash.core.lazy_imports import import_pyarrow + pa = import_pyarrow() + return pa.struct([pa.field(name, native) for name, native in inner_pairs]) + if target is TypeTarget.IBIS: + inner_str = ", ".join(f"{name}: {native}" for name, native in inner_pairs) + return f"struct<{inner_str}>" + return bare_native + + def _resolve_list_inner( field_name: str, item_type_str: str, target: TypeTarget, bare_native: Any ) -> Any: @@ -120,22 +147,12 @@ def to_polars_schema(schema: TypeSpec) -> Dict[str, Any]: {'id': Int64, 'name': Utf8} """ from mountainash.core.lazy_imports import import_polars - from mountainash.typespec._categorical import categorical_values pl = import_polars() if pl is None: raise ImportError("polars is required for to_polars_schema()") result = {} for f in schema.fields: - if f.categories is not None: - # categories takes priority over backend_type/type entirely - # (mirrors conform stage 5's branch order exactly). - values = categorical_values(f.categories) - result[f.name] = ( - pl.Enum([str(v) for v in values]) if f.categories_ordered - else pl.Categorical - ) - else: - result[f.name] = _resolve_field_native(f, TypeTarget.POLARS) + result[f.name] = _resolve_field_native(f, TypeTarget.POLARS) return result @@ -164,17 +181,9 @@ def to_pandas_dtypes(schema: TypeSpec) -> Dict[str, Any]: >>> pandas_dtypes {'id': 'Int64', 'name': 'string'} """ - from mountainash.typespec._categorical import categorical_values result: Dict[str, Any] = {} for f in schema.fields: - if f.categories is not None: - values = categorical_values(f.categories) - import pandas as pd - result[f.name] = pd.CategoricalDtype( - categories=values, ordered=bool(f.categories_ordered) - ) - else: - result[f.name] = _resolve_field_native(f, TypeTarget.PANDAS) + result[f.name] = _resolve_field_native(f, TypeTarget.PANDAS) return result diff --git a/src/mountainash/typespec/extraction.py b/src/mountainash/typespec/extraction.py index 012799db..44eddce8 100644 --- a/src/mountainash/typespec/extraction.py +++ b/src/mountainash/typespec/extraction.py @@ -430,6 +430,27 @@ def extract_from_dataframe( from_dataframe = extract_from_dataframe +def _fields_from_polars_struct(dtype: "pl.Struct") -> list["FieldSpec"]: + """Recursively build FieldSpec.object_fields from a Polars struct.""" + from mountainash.core.lazy_imports import import_polars + pl = import_polars() + fields = [] + for f in dtype.fields: + name, inner = f.name, f.dtype + item_type = None + if isinstance(inner, pl.List) and inner.inner is not None: + inner_universal, _ = from_canonical( + registry.from_native(inner.inner, target=TypeTarget.POLARS) + ) + item_type = inner_universal.value + object_fields = _fields_from_polars_struct(inner) if isinstance(inner, pl.Struct) else None + universal_type, _ = from_canonical(registry.from_native(inner, target=TypeTarget.POLARS)) + fields.append(FieldSpec( + name=name, type=universal_type, item_type=item_type, object_fields=object_fields, + )) + return fields + + def _from_polars(df: 'pl.DataFrame', preserve_backend_types: bool, **metadata) -> TypeSpec: """Extract schema from Polars DataFrame or LazyFrame.""" from mountainash.core.lazy_imports import import_polars @@ -443,23 +464,24 @@ def _from_polars(df: 'pl.DataFrame', preserve_backend_types: bool, **metadata) - schema_dict = df.schema for col_name, dtype in schema_dict.items(): - # Get backend type name - backend_type_str = str(dtype) - - # Convert to universal type universal_type = _universal_from_native(dtype, TypeTarget.POLARS) item_type = None + object_fields = None if isinstance(dtype, pl.List) and dtype.inner is not None: inner_universal, _ = from_canonical( registry.from_native(dtype.inner, target=TypeTarget.POLARS) ) item_type = inner_universal.value + elif isinstance(dtype, pl.Struct): + object_fields = _fields_from_polars_struct(dtype) + backend_type_str = None if isinstance(dtype, pl.Struct) else str(dtype) schema_field = FieldSpec( name=col_name, type=universal_type, item_type=item_type, + object_fields=object_fields, backend_type=backend_type_str if preserve_backend_types else None, ) fields.append(schema_field) @@ -505,6 +527,27 @@ def _from_pandas(df: 'pd.DataFrame', preserve_backend_types: bool, **metadata) - ) +def _fields_from_pyarrow_struct(dtype: "pa.StructType") -> list["FieldSpec"]: + """Recursively build FieldSpec.object_fields from a PyArrow struct.""" + from mountainash.core.lazy_imports import import_pyarrow + pa = import_pyarrow() + fields = [] + for f in dtype: + name, inner = f.name, f.type + item_type = None + if pa.types.is_list(inner): + inner_universal, _ = from_canonical( + registry.from_native(inner.value_type, target=TypeTarget.PYARROW) + ) + item_type = inner_universal.value + object_fields = _fields_from_pyarrow_struct(inner) if pa.types.is_struct(inner) else None + universal_type, _ = from_canonical(registry.from_native(inner, target=TypeTarget.PYARROW)) + fields.append(FieldSpec( + name=name, type=universal_type, item_type=item_type, object_fields=object_fields, + )) + return fields + + def _from_pyarrow(table: 'pa.Table', preserve_backend_types: bool, **metadata) -> TypeSpec: """Extract schema from PyArrow Table.""" from mountainash.core.lazy_imports import import_pyarrow @@ -515,26 +558,25 @@ def _from_pyarrow(table: 'pa.Table', preserve_backend_types: bool, **metadata) - fields = [] for field in table.schema: - # Get backend type backend_type = field.type - - # Convert to string for normalization - backend_type_str = str(backend_type) - - # Convert to universal type universal_type = _universal_from_native(backend_type, TypeTarget.PYARROW) item_type = None + object_fields = None if pa.types.is_list(backend_type): inner_universal, _ = from_canonical( registry.from_native(backend_type.value_type, target=TypeTarget.PYARROW) ) item_type = inner_universal.value + elif pa.types.is_struct(backend_type): + object_fields = _fields_from_pyarrow_struct(backend_type) + backend_type_str = None if pa.types.is_struct(backend_type) else str(backend_type) schema_field = FieldSpec( name=field.name, type=universal_type, item_type=item_type, + object_fields=object_fields, backend_type=backend_type_str if preserve_backend_types else None, ) fields.append(schema_field) diff --git a/src/mountainash/typespec/frictionless.py b/src/mountainash/typespec/frictionless.py index a23696e0..090d7a5d 100644 --- a/src/mountainash/typespec/frictionless.py +++ b/src/mountainash/typespec/frictionless.py @@ -102,6 +102,73 @@ def _parse_constraints( # Export: TypeSpec → Frictionless dict # --------------------------------------------------------------------------- +def _field_to_frictionless_dict(fspec: "FieldSpec") -> Dict[str, Any]: + """Export one FieldSpec to a Frictionless field descriptor dict. + + Recurses into ``object_fields`` (item 102) so nested fields retain the + complete field descriptor shape of top-level fields. + """ + field_dict: Dict[str, Any] = { + "name": fspec.name, + "type": fspec.type.value if isinstance(fspec.type, UniversalType) else str(fspec.type), + } + + if fspec.format != "default": + field_dict["format"] = fspec.format + if fspec.title: + field_dict["title"] = fspec.title + if fspec.description: + field_dict["description"] = fspec.description + if fspec.constraints is not None: + constraints_dict = _constraints_to_dict(fspec.constraints) + if constraints_dict: + field_dict["constraints"] = constraints_dict + if fspec.missing_values is not None: + field_dict["missingValues"] = fspec.missing_values + if fspec.categories is not None: + field_dict["categories"] = fspec.categories + if fspec.true_values is not None: + field_dict["trueValues"] = fspec.true_values + if fspec.false_values is not None: + field_dict["falseValues"] = fspec.false_values + if fspec.categories_ordered is not None: + field_dict["categoriesOrdered"] = fspec.categories_ordered + if fspec.example is not None: + field_dict["example"] = fspec.example + if fspec.rdf_type is not None: + field_dict["rdfType"] = fspec.rdf_type + if fspec.decimal_char is not None: + field_dict["decimalChar"] = fspec.decimal_char + if fspec.group_char is not None: + field_dict["groupChar"] = fspec.group_char + if fspec.bare_number is not None: + field_dict["bareNumber"] = fspec.bare_number + if fspec.item_type is not None: + field_dict["itemType"] = fspec.item_type + if fspec.delimiter is not None: + field_dict["delimiter"] = fspec.delimiter + + field_extensions: Dict[str, Any] = {} + if fspec.rename_from is not None: + field_extensions["rename_from"] = fspec.rename_from + if fspec.null_fill is not None: + field_extensions["null_fill"] = fspec.null_fill + if fspec.custom_cast is not None: + field_extensions["custom_cast"] = fspec.custom_cast + if fspec.constraints and fspec.constraints.enum_weights is not None: + field_extensions["enum_weights"] = fspec.constraints.enum_weights + if fspec.backend_type is not None: + field_extensions["backend_type"] = fspec.backend_type + if fspec.object_fields is not None: + field_extensions["object_fields"] = [ + _field_to_frictionless_dict(inner) for inner in fspec.object_fields + ] + if field_extensions: + field_dict["x-mountainash"] = field_extensions + + return field_dict + + def typespec_to_frictionless(spec: TypeSpec) -> Dict[str, Any]: """Convert a TypeSpec to a Frictionless Table Schema descriptor dict. @@ -153,64 +220,9 @@ def typespec_to_frictionless(spec: TypeSpec) -> Dict[str, Any]: descriptor["x-mountainash"] = spec_ext # Fields - fields_list: List[Dict[str, Any]] = [] - for fspec in spec.fields: - field_dict: Dict[str, Any] = { - "name": fspec.name, - "type": fspec.type.value if isinstance(fspec.type, UniversalType) else str(fspec.type), - } - - if fspec.format != "default": - field_dict["format"] = fspec.format - if fspec.title: - field_dict["title"] = fspec.title - if fspec.description: - field_dict["description"] = fspec.description - if fspec.constraints is not None: - constraints_dict = _constraints_to_dict(fspec.constraints) - if constraints_dict: - field_dict["constraints"] = constraints_dict - if fspec.missing_values is not None: - field_dict["missingValues"] = fspec.missing_values - if fspec.categories is not None: # Gap 7 - field_dict["categories"] = fspec.categories - if fspec.true_values is not None: # Gap 9 - field_dict["trueValues"] = fspec.true_values - if fspec.false_values is not None: # Gap 9 - field_dict["falseValues"] = fspec.false_values - if fspec.categories_ordered is not None: - field_dict["categoriesOrdered"] = fspec.categories_ordered - if fspec.example is not None: - field_dict["example"] = fspec.example - if fspec.rdf_type is not None: - field_dict["rdfType"] = fspec.rdf_type - if fspec.decimal_char is not None: - field_dict["decimalChar"] = fspec.decimal_char - if fspec.group_char is not None: - field_dict["groupChar"] = fspec.group_char - if fspec.bare_number is not None: - field_dict["bareNumber"] = fspec.bare_number - if fspec.item_type is not None: - field_dict["itemType"] = fspec.item_type - if fspec.delimiter is not None: - field_dict["delimiter"] = fspec.delimiter - - # Field-level x-mountainash extensions - field_extensions: Dict[str, Any] = {} - if fspec.rename_from is not None: - field_extensions["rename_from"] = fspec.rename_from - if fspec.null_fill is not None: - field_extensions["null_fill"] = fspec.null_fill - if fspec.custom_cast is not None: - field_extensions["custom_cast"] = fspec.custom_cast - if fspec.constraints and fspec.constraints.enum_weights is not None: - field_extensions["enum_weights"] = fspec.constraints.enum_weights - if fspec.backend_type is not None: # Gap 10: move under x-mountainash - field_extensions["backend_type"] = fspec.backend_type - if field_extensions: - field_dict["x-mountainash"] = field_extensions - - fields_list.append(field_dict) + fields_list: List[Dict[str, Any]] = [ + _field_to_frictionless_dict(fspec) for fspec in spec.fields + ] descriptor["fields"] = fields_list return descriptor @@ -220,6 +232,73 @@ def typespec_to_frictionless(spec: TypeSpec) -> Dict[str, Any]: # Import: Frictionless dict / JSON string / Path → TypeSpec # --------------------------------------------------------------------------- +def _field_from_frictionless_dict(raw_field: Dict[str, Any]) -> "FieldSpec": + """Import one Frictionless field descriptor into a FieldSpec. + + Recurses into ``x-mountainash.object_fields`` (item 102) so nested + descriptors mirror the export helper exactly. + """ + name: str = raw_field["name"] + type_str: str = raw_field.get("type", "string") + universal_type = parse_universal(type_str) + + format_: str = raw_field.get("format", "default") + field_title: Optional[str] = raw_field.get("title") + field_description: Optional[str] = raw_field.get("description") + field_missing_values: Optional[List[str]] = raw_field.get("missingValues") + categories: Optional[List[Any]] = raw_field.get("categories") + true_values: Optional[List[str]] = raw_field.get("trueValues") + false_values: Optional[List[str]] = raw_field.get("falseValues") + categories_ordered: Optional[bool] = raw_field.get("categoriesOrdered") + example: Optional[Any] = raw_field.get("example") + rdf_type: Optional[str] = raw_field.get("rdfType") + decimal_char: Optional[str] = raw_field.get("decimalChar") + group_char: Optional[str] = raw_field.get("groupChar") + bare_number: Optional[bool] = raw_field.get("bareNumber") + item_type: Optional[str] = raw_field.get("itemType") + delimiter: Optional[str] = raw_field.get("delimiter") + + field_ext: Dict[str, Any] = raw_field.get("x-mountainash", {}) or {} + rename_from: Optional[str] = field_ext.get("rename_from") + null_fill: Any = field_ext.get("null_fill") + custom_cast: Optional[str] = field_ext.get("custom_cast") + enum_weights: Optional[Dict[str, float]] = field_ext.get("enum_weights") + backend_type: Optional[str] = field_ext.get("backend_type") + raw_object_fields: Optional[List[Dict[str, Any]]] = field_ext.get("object_fields") + object_fields: Optional[List[FieldSpec]] = ( + [_field_from_frictionless_dict(rf) for rf in raw_object_fields] + if raw_object_fields is not None else None + ) + + constraints = _parse_constraints(raw_field.get("constraints"), enum_weights=enum_weights) + + return FieldSpec( + name=name, + type=universal_type, + format=format_, + title=field_title, + description=field_description, + constraints=constraints, + missing_values=field_missing_values, + categories=categories, + categories_ordered=categories_ordered, + example=example, + rdf_type=rdf_type, + decimal_char=decimal_char, + group_char=group_char, + bare_number=bare_number, + item_type=item_type, + delimiter=delimiter, + true_values=true_values, + false_values=false_values, + backend_type=backend_type, + rename_from=rename_from, + null_fill=null_fill, + custom_cast=custom_cast, + object_fields=object_fields, + ) + + def typespec_from_frictionless(data: Union[Dict[str, Any], str, Path]) -> TypeSpec: """Create a TypeSpec from a Frictionless Table Schema descriptor. @@ -280,64 +359,10 @@ def typespec_from_frictionless(data: Union[Dict[str, Any], str, Path]) -> TypeSp foreign_keys = [foreign_key_from_dict(raw_fk) for raw_fk in raw_fks] # -- Fields -- - fields: List[FieldSpec] = [] - for raw_field in descriptor.get("fields", []): - name: str = raw_field["name"] - type_str: str = raw_field.get("type", "string") - universal_type = parse_universal(type_str) - - format_: str = raw_field.get("format", "default") - field_title: Optional[str] = raw_field.get("title") - field_description: Optional[str] = raw_field.get("description") - field_missing_values: Optional[List[str]] = raw_field.get("missingValues") - categories: Optional[List[Any]] = raw_field.get("categories") # Gap 7 - true_values: Optional[List[str]] = raw_field.get("trueValues") # Gap 9 - false_values: Optional[List[str]] = raw_field.get("falseValues") # Gap 9 - categories_ordered: Optional[bool] = raw_field.get("categoriesOrdered") - example: Optional[Any] = raw_field.get("example") - rdf_type: Optional[str] = raw_field.get("rdfType") - decimal_char: Optional[str] = raw_field.get("decimalChar") - group_char: Optional[str] = raw_field.get("groupChar") - bare_number: Optional[bool] = raw_field.get("bareNumber") - item_type: Optional[str] = raw_field.get("itemType") - delimiter: Optional[str] = raw_field.get("delimiter") - - # Field-level x-mountainash extensions - field_ext: Dict[str, Any] = raw_field.get("x-mountainash", {}) or {} - rename_from: Optional[str] = field_ext.get("rename_from") - null_fill: Any = field_ext.get("null_fill") - custom_cast: Optional[str] = field_ext.get("custom_cast") - enum_weights: Optional[Dict[str, float]] = field_ext.get("enum_weights") - backend_type: Optional[str] = field_ext.get("backend_type") # Gap 10 - - constraints = _parse_constraints(raw_field.get("constraints"), enum_weights=enum_weights) - - fields.append( - FieldSpec( - name=name, - type=universal_type, - format=format_, - title=field_title, - description=field_description, - constraints=constraints, - missing_values=field_missing_values, - categories=categories, - categories_ordered=categories_ordered, - example=example, - rdf_type=rdf_type, - decimal_char=decimal_char, - group_char=group_char, - bare_number=bare_number, - item_type=item_type, - delimiter=delimiter, - true_values=true_values, - false_values=false_values, - backend_type=backend_type, - rename_from=rename_from, - null_fill=null_fill, - custom_cast=custom_cast, - ) - ) + fields: List[FieldSpec] = [ + _field_from_frictionless_dict(raw_field) + for raw_field in descriptor.get("fields", []) + ] return TypeSpec( fields=fields, diff --git a/src/mountainash/typespec/spec.py b/src/mountainash/typespec/spec.py index 0319d027..bf62f59b 100644 --- a/src/mountainash/typespec/spec.py +++ b/src/mountainash/typespec/spec.py @@ -89,6 +89,7 @@ class FieldSpec: group_char: Optional[str] = None bare_number: Optional[bool] = None item_type: Optional[str] = None + object_fields: Optional[List["FieldSpec"]] = None # x-mountainash: OBJECT inner-field schema delimiter: Optional[str] = None backend_type: Optional[str] = None null_fill: Any = None @@ -136,6 +137,8 @@ def to_dict(self) -> Dict[str, Any]: result["bareNumber"] = self.bare_number if self.item_type is not None: result["itemType"] = self.item_type + if self.object_fields is not None: + result["objectFields"] = [f.to_dict() for f in self.object_fields] if self.delimiter is not None: result["delimiter"] = self.delimiter if self.backend_type: diff --git a/tests/_spine_expectation_census.md b/tests/_spine_expectation_census.md index 45932b46..0a9445a8 100644 --- a/tests/_spine_expectation_census.md +++ b/tests/_spine_expectation_census.md @@ -20,6 +20,7 @@ Buckets: `migrated` (derivable from the spine today), `retained` (a LITERAL_ONLY | tests/conform/cross_backend/test_conform_transforms.py:14 | static-marker | UNRESOLVED | UNRESOLVED | UNRESOLVED | None | spine-derived id-keyed divergence mark via xfail_divergence('MA-CONF-03') — migrated | | tests/conform/cross_backend/test_null_fill_mode.py:66 | static-marker | UNRESOLVED | UNRESOLVED | UNRESOLVED | None | spine-derived id-keyed divergence mark via xfail_divergence('MA-TYPE-02') — migrated | | tests/conform/cross_backend/test_relation_conform.py:13 | static-marker | UNRESOLVED | UNRESOLVED | UNRESOLVED | None | spine-derived id-keyed divergence mark via xfail_divergence('MA-CONF-01') — migrated | +| tests/conform/cross_backend/test_relation_conform.py:141 | static-marker | UNRESOLVED | ibis-sqlite | UNRESOLVED | None | spine-derived id-keyed divergence mark via xfail_divergence('MA-CONF-04') — migrated | | tests/core/test_capability_gating.py:228 | static-marker | UNRESOLVED | ibis-duckdb | UNRESOLVED | None | spine-derived id-keyed divergence mark via xfail_divergence('IB-TYPE-02') — migrated | | tests/core/test_capability_gating.py:236 | static-marker | UNRESOLVED | ibis-duckdb | UNRESOLVED | None | spine-derived id-keyed divergence mark via xfail_divergence('MA-MATH-01') — migrated | | tests/core/test_capability_gating.py:241 | static-marker | UNRESOLVED | ibis-duckdb | UNRESOLVED | None | spine-derived id-keyed divergence mark via xfail_divergence('IB-TYPE-02') — migrated | diff --git a/tests/conform/cross_backend/test_relation_conform.py b/tests/conform/cross_backend/test_relation_conform.py index a71a8250..62c8602b 100644 --- a/tests/conform/cross_backend/test_relation_conform.py +++ b/tests/conform/cross_backend/test_relation_conform.py @@ -134,6 +134,33 @@ def test_subset_dotted_source_extracts_with_extra_column(self, backend_name, bac # but discarded from the OUTPUT. assert list(result.columns) == ["pid"] assert "other" not in result.columns +_STRUCT_CAST = [ + pytest.param("narwhals-polars"), + pytest.param("ibis-polars"), + pytest.param("ibis-duckdb"), + pytest.param("ibis-sqlite", marks=xfail_divergence("MA-CONF-04", backend="ibis-sqlite")), + # narwhals-pandas deliberately excluded: MA-CONF-01 already establishes + # pandas-backed struct data is broken at a more fundamental level. +] + + +@pytest.mark.parametrize("backend_name", _STRUCT_CAST) +class TestRelationConformStructCastOffPolars: + """Struct conform materialization is deliberately Polars-only.""" + + def test_struct_cast_raises_dtype_mapping_error_off_polars(self, backend_name, backend_factory): + from mountainash.core.dtypes import DtypeMappingError + df = backend_factory.create({"addr": [{"street": "Main St", "zip": "12345"}]}, backend_name) + spec = TypeSpec( + fields=[FieldSpec(name="addr", type=UniversalType.OBJECT, object_fields=[ + FieldSpec(name="street", type=UniversalType.STRING), + FieldSpec(name="zip", type=UniversalType.STRING), + ])], + ) + with pytest.raises(DtypeMappingError): + ma.relation(df).conform(spec).to_polars() + + class TestRelationConformFullPipeline: diff --git a/tests/conform/test_struct_casting.py b/tests/conform/test_struct_casting.py new file mode 100644 index 00000000..f8d5a58a --- /dev/null +++ b/tests/conform/test_struct_casting.py @@ -0,0 +1,99 @@ +"""Tests for nested STRUCT (object_fields) casting in the conform pipeline +(item 102, spec §5). Polars-only — see spec §5's Consequence and this +plan's Global Constraints; the cross-backend DtypeMappingError companion +assertion lives in tests/conform/cross_backend/test_relation_conform.py. +""" +from __future__ import annotations + +import polars as pl +import pytest + +import mountainash as ma +from mountainash.typespec.spec import FieldSpec, TypeSpec +from mountainash.typespec.universal_types import UniversalType + + +class TestStructCastingFlat: + def test_flat_struct_cast_by_name(self): + df = pl.DataFrame([{"addr": {"street": "Main St", "zip": "12345"}}]) + spec = TypeSpec(fields=[ + FieldSpec(name="addr", type=UniversalType.OBJECT, object_fields=[ + FieldSpec(name="street", type=UniversalType.STRING), + FieldSpec(name="zip", type=UniversalType.STRING), + ]), + ]) + result = ma.relation(df).conform(spec).to_polars() + assert result.schema["addr"] == pl.Struct({"street": pl.String, "zip": pl.String}) + assert result["addr"].to_list() == [{"street": "Main St", "zip": "12345"}] + + def test_struct_field_matching_is_by_name_not_position(self): + """Spec §5.2 spike (§13.1): reordered targets match source by name.""" + df = pl.DataFrame([{"addr": {"street": "Main St", "zip": "12345"}}]) + spec = TypeSpec(fields=[ + FieldSpec(name="addr", type=UniversalType.OBJECT, object_fields=[ + FieldSpec(name="zip", type=UniversalType.STRING), + FieldSpec(name="street", type=UniversalType.STRING), + ]), + ]) + result = ma.relation(df).conform(spec).to_polars() + row = result["addr"].to_list()[0] + assert row["zip"] == "12345" + assert row["street"] == "Main St" + + def test_inner_type_cast_applied(self): + """Inner fields are cast, not only structurally wrapped.""" + df = pl.DataFrame([{"stats": {"count": "3", "label": "x"}}]) + spec = TypeSpec(fields=[ + FieldSpec(name="stats", type=UniversalType.OBJECT, object_fields=[ + FieldSpec(name="count", type=UniversalType.INTEGER), + FieldSpec(name="label", type=UniversalType.STRING), + ]), + ]) + result = ma.relation(df).conform(spec).to_polars() + assert {f.name: f.dtype for f in result.schema["stats"].fields}["count"] == pl.Int64 + assert result["stats"].to_list() == [{"count": 3, "label": "x"}] + + +class TestStructCastingNested: + def test_two_level_nested_struct_cast(self): + df = pl.DataFrame([{"addr": {"street": "Main St", "geo": {"lat": "1.5", "lon": "2.5"}}}]) + spec = TypeSpec(fields=[ + FieldSpec(name="addr", type=UniversalType.OBJECT, object_fields=[ + FieldSpec(name="street", type=UniversalType.STRING), + FieldSpec(name="geo", type=UniversalType.OBJECT, object_fields=[ + FieldSpec(name="lat", type=UniversalType.NUMBER), + FieldSpec(name="lon", type=UniversalType.NUMBER), + ]), + ]), + ]) + result = ma.relation(df).conform(spec).to_polars() + expected_dtype = pl.Struct({ + "street": pl.String, + "geo": pl.Struct({"lat": pl.Float64, "lon": pl.Float64}), + }) + assert result.schema["addr"] == expected_dtype + assert result["addr"].to_list() == [{"street": "Main St", "geo": {"lat": 1.5, "lon": 2.5}}] + + +class TestStructCastingErrorHandling: + def test_incompatible_inner_type_raises(self): + """Spec §6/§13.1: incompatible inner type raises.""" + df = pl.DataFrame([{"addr": {"count": "not-a-number"}}]) + spec = TypeSpec(fields=[ + FieldSpec(name="addr", type=UniversalType.OBJECT, object_fields=[ + FieldSpec(name="count", type=UniversalType.INTEGER), + ]), + ]) + with pytest.raises(Exception): + ma.relation(df).conform(spec).to_polars() + + def test_categories_wins_over_object_fields_on_same_field(self): + """Categories precede OBJECT dispatch when both are declared.""" + df = pl.DataFrame({"x": ["a", "b"]}) + spec = TypeSpec(fields=[ + FieldSpec(name="x", type=UniversalType.OBJECT, + categories=["a", "b"], categories_ordered=False, + object_fields=[FieldSpec(name="never", type=UniversalType.STRING)]), + ]) + result = ma.relation(df).conform(spec).to_polars() + assert result.schema["x"] == pl.Categorical diff --git a/tests/core/test_divergence_facts.py b/tests/core/test_divergence_facts.py index bc603110..89215442 100644 --- a/tests/core/test_divergence_facts.py +++ b/tests/core/test_divergence_facts.py @@ -20,6 +20,10 @@ def test_lookup_and_unknown_id(): assert divergence_by_id(some.id) is some with pytest.raises(KeyError, match="unknown divergence id"): divergence_by_id("XX-NOPE-99") +def test_ma_conf_04_registered(): + fact = divergence_by_id("MA-CONF-04") + assert fact is not None + assert fact.backends == ("ibis-sqlite",) def test_narwhals_pandas_titlecase_divergence_registered(): diff --git a/tests/typespec/test_converters.py b/tests/typespec/test_converters.py index ac0d914b..35e3f4e8 100644 --- a/tests/typespec/test_converters.py +++ b/tests/typespec/test_converters.py @@ -368,6 +368,126 @@ def test_unknown_item_type_raises_with_field_context_and_chain(self): # chain: __cause__ is the original UnknownDtypeError parse_universal # raised, not swallowed by a message-only copy assert isinstance(exc_info.value.__cause__, UnknownDtypeError) +# ============================================================================ +# TestNestedStructObjectFields (item 102) +# ============================================================================ + +class TestNestedStructObjectFields: + """object_fields: nested STRUCT inner-field schema via FieldSpec.""" + + def _spec(self, object_fields=None): + return TypeSpec(fields=[ + FieldSpec(name="addr", type=UniversalType.OBJECT, object_fields=object_fields), + ]) + + def _flat_fields(self): + return [ + FieldSpec(name="street", type=UniversalType.STRING), + FieldSpec(name="zip", type=UniversalType.STRING), + ] + + def _nested_fields(self): + return [ + FieldSpec(name="street", type=UniversalType.STRING), + FieldSpec(name="geo", type=UniversalType.OBJECT, object_fields=[ + FieldSpec(name="lat", type=UniversalType.NUMBER), + FieldSpec(name="lon", type=UniversalType.NUMBER), + ]), + ] + + def test_polars_flat_struct_resolves_inner_fields(self): + import polars as pl + result = to_polars_schema(self._spec(self._flat_fields())) + assert result["addr"] == pl.Struct({"street": pl.String, "zip": pl.String}) + + def test_polars_two_level_nested_struct_resolves_recursively(self): + import polars as pl + result = to_polars_schema(self._spec(self._nested_fields())) + expected = pl.Struct({ + "street": pl.String, + "geo": pl.Struct({"lat": pl.Float64, "lon": pl.Float64}), + }) + assert result["addr"] == expected + + def test_pyarrow_flat_struct_resolves_inner_fields(self): + pytest.importorskip("pyarrow") + import pyarrow as pa + result = to_arrow_schema(self._spec(self._flat_fields())) + field = result.field("addr") + assert pa.types.is_struct(field.type) + assert field.type.field("street").type == pa.string() + assert field.type.field("zip").type == pa.string() + + def test_narwhals_flat_struct_resolves_inner_fields(self): + import narwhals as nw + from mountainash.core.dtypes import TypeTarget + from mountainash.typespec.converters import _resolve_field_native + native = _resolve_field_native(self._spec(self._flat_fields()).fields[0], TypeTarget.NARWHALS) + assert native == nw.Struct({"street": nw.String, "zip": nw.String}) + + def test_ibis_flat_struct_resolves_inner_fields_as_schema_string(self): + result = to_ibis_schema(self._spec(self._flat_fields())) + assert result["addr"] == "struct" + + def test_ibis_two_level_nested_struct_schema_string(self): + result = to_ibis_schema(self._spec(self._nested_fields())) + assert result["addr"] == "struct>" + + def test_pandas_stays_object_regardless_of_object_fields(self): + result = to_pandas_dtypes(self._spec(self._flat_fields())) + assert result["addr"] == "object" + + def test_no_object_fields_keeps_bare_container(self): + import polars as pl + result = to_polars_schema(self._spec(None)) + assert result["addr"] is pl.Struct + + def test_empty_object_fields_list_keeps_bare_container(self): + import polars as pl + result = to_polars_schema(self._spec([])) + assert result["addr"] is pl.Struct + + def test_inner_field_categories_resolve_correctly(self): + import polars as pl + fields = [FieldSpec(name="kind", type=UniversalType.STRING, + categories=["home", "work"], categories_ordered=True)] + result = to_polars_schema(self._spec(fields)) + assert result["addr"] == pl.Struct({"kind": pl.Enum(["home", "work"])}) + + def test_recursion_error_on_pathologically_deep_chain(self): + from mountainash.core.dtypes import TypeTarget + from mountainash.typespec.converters import _resolve_struct_inner + deep = FieldSpec(name="leaf", type=UniversalType.INTEGER) + for i in range(10000): + deep = FieldSpec(name=f"level_{i}", type=UniversalType.OBJECT, object_fields=[deep]) + with pytest.raises(RecursionError): + _resolve_struct_inner("addr", [deep], TypeTarget.POLARS, None) + + +# ============================================================================ +# TestCategoricalRefactorRegression (item 102, spec §4.1) +# ============================================================================ + +class TestCategoricalRefactorRegression: + def test_resolve_field_native_polars_categorical(self): + import polars as pl + from mountainash.core.dtypes import TypeTarget + from mountainash.typespec.converters import _resolve_field_native + field = FieldSpec(name="cat", type=UniversalType.STRING, + categories=["a", "b"], categories_ordered=False) + assert _resolve_field_native(field, TypeTarget.POLARS) is pl.Categorical + + def test_resolve_field_native_pandas_categorical(self): + import pandas as pd + from mountainash.core.dtypes import TypeTarget + from mountainash.typespec.converters import _resolve_field_native + field = FieldSpec(name="cat", type=UniversalType.STRING, + categories=["a", "b"], categories_ordered=True) + result = _resolve_field_native(field, TypeTarget.PANDAS) + assert isinstance(result, pd.CategoricalDtype) + assert list(result.categories) == ["a", "b"] + assert result.ordered is True + # ============================================================================ diff --git a/tests/typespec/test_extraction.py b/tests/typespec/test_extraction.py index 39df5c13..9a3fb450 100644 --- a/tests/typespec/test_extraction.py +++ b/tests/typespec/test_extraction.py @@ -398,6 +398,83 @@ def test_polars_list_populates_item_type(self): assert field.type == UniversalType.ARRAY assert field.item_type == "integer" + def test_polars_struct_populates_object_fields(self): + import polars as pl + from mountainash.typespec.extraction import extract_from_dataframe + from mountainash.typespec.universal_types import UniversalType + df = pl.DataFrame([{"addr": {"street": "Main St", "zip": "12345"}}]) + spec = extract_from_dataframe(df) + (field,) = [f for f in spec.fields if f.name == "addr"] + assert field.type == UniversalType.OBJECT + names = {f.name: f.type for f in field.object_fields} + assert names == {"street": UniversalType.STRING, "zip": UniversalType.STRING} + + def test_polars_struct_two_levels_deep_populates_object_fields_recursively(self): + import polars as pl + from mountainash.typespec.extraction import extract_from_dataframe + from mountainash.typespec.universal_types import UniversalType + df = pl.DataFrame([{"addr": {"street": "Main St", "geo": {"lat": 1.0, "lon": 2.0}}}]) + spec = extract_from_dataframe(df) + (field,) = [f for f in spec.fields if f.name == "addr"] + (geo,) = [f for f in field.object_fields if f.name == "geo"] + assert geo.type == UniversalType.OBJECT + assert {f.name for f in geo.object_fields} == {"lat", "lon"} + + def test_polars_struct_extracts_with_backend_type_none(self): + import polars as pl + from mountainash.typespec.extraction import extract_from_dataframe + df = pl.DataFrame([{"addr": {"street": "Main St"}}]) + spec = extract_from_dataframe(df, preserve_backend_types=True) + (field,) = [f for f in spec.fields if f.name == "addr"] + assert field.backend_type is None + + def test_pyarrow_struct_populates_object_fields(self): + pytest.importorskip("pyarrow") + import pyarrow as pa + from mountainash.typespec.extraction import extract_from_dataframe + from mountainash.typespec.universal_types import UniversalType + table = pa.table({"addr": pa.array( + [{"street": "Main St", "zip": "12345"}], + type=pa.struct([pa.field("street", pa.string()), pa.field("zip", pa.string())]), + )}) + spec = extract_from_dataframe(table) + (field,) = [f for f in spec.fields if f.name == "addr"] + assert field.type == UniversalType.OBJECT + names = {f.name: f.type for f in field.object_fields} + assert names == {"street": UniversalType.STRING, "zip": UniversalType.STRING} + + def test_pyarrow_struct_extracts_with_backend_type_none(self): + pytest.importorskip("pyarrow") + import pyarrow as pa + from mountainash.typespec.extraction import extract_from_dataframe + table = pa.table({"addr": pa.array( + [{"street": "Main St"}], type=pa.struct([pa.field("street", pa.string())]), + )}) + spec = extract_from_dataframe(table, preserve_backend_types=True) + (field,) = [f for f in spec.fields if f.name == "addr"] + assert field.backend_type is None + + def test_extraction_resolver_round_trip_pins_full_struct_not_just_no_raise(self): + import polars as pl + from mountainash.typespec.extraction import extract_from_dataframe + from mountainash.typespec.converters import to_polars_schema + df = pl.DataFrame([{"addr": {"street": "Main St", "zip": "12345"}}]) + spec = extract_from_dataframe(df, preserve_backend_types=True) + result = to_polars_schema(spec) + assert result["addr"] == pl.Struct({"street": pl.String, "zip": pl.String}) + + def test_extraction_resolver_round_trip_empty_frame(self): + import polars as pl + from mountainash.typespec.extraction import extract_from_dataframe + from mountainash.relations.backends.relation_systems.polars.extensions_mountainash.relsys_pl_ext_ma_util import ( + MountainashPolarsExtensionRelationSystem, + ) + df = pl.DataFrame([{"addr": {"street": "Main St", "zip": "12345"}}]) + spec = extract_from_dataframe(df, preserve_backend_types=True) + empty = MountainashPolarsExtensionRelationSystem().empty_frame(spec) + collected = empty.collect() + assert collected.schema["addr"] == pl.Struct({"street": pl.String, "zip": pl.String}) + assert collected.shape == (0, 1) def test_unknown_dtype_raises(self): import polars as pl from mountainash.core.dtypes.errors import UnknownDtypeError diff --git a/tests/typespec/test_frictionless.py b/tests/typespec/test_frictionless.py index dc08bda7..2c3a8b99 100644 --- a/tests/typespec/test_frictionless.py +++ b/tests/typespec/test_frictionless.py @@ -127,6 +127,49 @@ def test_enum_weights_exported_in_extensions(self): field_dict = result["fields"][0] assert field_dict["constraints"]["enum"] == ["A", "B"] assert field_dict["x-mountainash"]["enum_weights"] == {"A": 0.7, "B": 0.3} + def test_object_fields_exported_under_x_mountainash(self): + spec = TypeSpec(fields=[ + FieldSpec(name="addr", type=UniversalType.OBJECT, object_fields=[ + FieldSpec(name="street", type=UniversalType.STRING), + FieldSpec(name="zip", type=UniversalType.STRING), + ]), + ]) + result = typespec_to_frictionless(spec) + field_result = result["fields"][0] + assert field_result["x-mountainash"]["object_fields"] == [ + {"name": "street", "type": "string"}, + {"name": "zip", "type": "string"}, + ] + + def test_object_fields_two_levels_deep_exported(self): + spec = TypeSpec(fields=[ + FieldSpec(name="addr", type=UniversalType.OBJECT, object_fields=[ + FieldSpec(name="geo", type=UniversalType.OBJECT, object_fields=[ + FieldSpec(name="lat", type=UniversalType.NUMBER), + FieldSpec(name="lon", type=UniversalType.NUMBER), + ]), + ]), + ]) + result = typespec_to_frictionless(spec) + geo = result["fields"][0]["x-mountainash"]["object_fields"][0] + assert geo["name"] == "geo" + assert geo["type"] == "object" + assert geo["x-mountainash"]["object_fields"] == [ + {"name": "lat", "type": "number"}, + {"name": "lon", "type": "number"}, + ] + + def test_nested_field_carries_its_own_categories(self): + """Nested object_fields entries are complete field descriptors.""" + spec = TypeSpec(fields=[ + FieldSpec(name="addr", type=UniversalType.OBJECT, object_fields=[ + FieldSpec(name="kind", type=UniversalType.STRING, + categories=["home", "work"], categories_ordered=False), + ]), + ]) + result = typespec_to_frictionless(spec) + inner = result["fields"][0]["x-mountainash"]["object_fields"][0] + assert inner["categories"] == ["home", "work"] # ============================================================================ @@ -270,6 +313,39 @@ def test_enum_weights_imported_from_extensions(self): } spec = typespec_from_frictionless(descriptor) assert spec.fields[0].constraints.enum_weights == {"A": 0.7, "B": 0.3} + def test_object_fields_imported_recursively(self): + descriptor = { + "fields": [ + { + "name": "addr", "type": "object", + "x-mountainash": { + "object_fields": [ + {"name": "street", "type": "string"}, + { + "name": "geo", "type": "object", + "x-mountainash": { + "object_fields": [ + {"name": "lat", "type": "number"}, + ] + }, + }, + ] + }, + } + ] + } + spec = typespec_from_frictionless(descriptor) + addr = spec.get_field("addr") + assert addr is not None + assert [f.name for f in addr.object_fields] == ["street", "geo"] + geo = addr.object_fields[1] + assert geo.object_fields[0].name == "lat" + assert geo.object_fields[0].type == UniversalType.NUMBER + + def test_no_object_fields_key_leaves_none(self): + descriptor = {"fields": [{"name": "x", "type": "string"}]} + spec = typespec_from_frictionless(descriptor) + assert spec.get_field("x").object_fields is None def test_foreign_keys_round_trip(self): fk = ForeignKey( @@ -344,6 +420,21 @@ def test_full_round_trip(self): score_field = reimported.get_field("score") assert score_field is not None assert score_field.null_fill == 0.0 + def test_object_fields_round_trip_two_levels_deep(self): + original = TypeSpec(fields=[ + FieldSpec(name="addr", type=UniversalType.OBJECT, object_fields=[ + FieldSpec(name="street", type=UniversalType.STRING), + FieldSpec(name="geo", type=UniversalType.OBJECT, object_fields=[ + FieldSpec(name="lat", type=UniversalType.NUMBER), + FieldSpec(name="lon", type=UniversalType.NUMBER), + ]), + ]), + ]) + descriptor = typespec_to_frictionless(original) + restored = typespec_from_frictionless(descriptor) + redescriptor = typespec_to_frictionless(restored) + assert redescriptor == descriptor + assert restored.get_field("addr").object_fields[1].object_fields[0].name == "lat" def test_json_serializable(self): spec = TypeSpec( diff --git a/tests/typespec/test_spec.py b/tests/typespec/test_spec.py index 045b3814..11cca4b9 100644 --- a/tests/typespec/test_spec.py +++ b/tests/typespec/test_spec.py @@ -73,6 +73,37 @@ def test_to_dict_with_constraints(self): assert "constraints" in d assert d["constraints"]["required"] is True assert d["constraints"]["min_length"] == 2 + def test_field_spec_has_object_fields_default_none(self): + f = FieldSpec(name="addr", type=UniversalType.OBJECT) + assert f.object_fields is None + + def test_object_fields_omitted_from_dict_when_unset(self): + f = FieldSpec(name="addr", type=UniversalType.OBJECT) + d = f.to_dict() + assert "objectFields" not in d + + def test_object_fields_exported_recursively_in_to_dict(self): + inner = [ + FieldSpec(name="street", type=UniversalType.STRING), + FieldSpec(name="zip", type=UniversalType.STRING), + ] + f = FieldSpec(name="addr", type=UniversalType.OBJECT, object_fields=inner) + d = f.to_dict() + assert d["objectFields"] == [ + {"name": "street", "type": "string"}, + {"name": "zip", "type": "string"}, + ] + + def test_object_fields_two_levels_deep_in_to_dict(self): + geo = [FieldSpec(name="lat", type=UniversalType.NUMBER)] + inner = [ + FieldSpec(name="street", type=UniversalType.STRING), + FieldSpec(name="geo", type=UniversalType.OBJECT, object_fields=geo), + ] + f = FieldSpec(name="addr", type=UniversalType.OBJECT, object_fields=inner) + d = f.to_dict() + assert d["objectFields"][1]["name"] == "geo" + assert d["objectFields"][1]["objectFields"] == [{"name": "lat", "type": "number"}] # ============================================================================