diff --git a/dascore/core/attrs.py b/dascore/core/attrs.py index e8ca3ab15..2f276e92f 100644 --- a/dascore/core/attrs.py +++ b/dascore/core/attrs.py @@ -19,11 +19,11 @@ DataType, max_lens, ) +from dascore.models import DascoreBaseModel, UnitQuantity from dascore.utils.misc import ( to_str, validate_acquisition_key, ) -from dascore.utils.models import DascoreBaseModel, UnitQuantity str_validator = PlainValidator(to_str) diff --git a/dascore/core/coordmanager.py b/dascore/core/coordmanager.py index f52873f6b..c61ccea9f 100644 --- a/dascore/core/coordmanager.py +++ b/dascore/core/coordmanager.py @@ -66,6 +66,12 @@ ParameterError, PatchBroadcastError, ) +from dascore.models import ( + ArrayLike, + DascoreBaseModel, + frozen_dict_serializer, + frozen_dict_validator, +) from dascore.utils.docs import compose_docstring from dascore.utils.mapping import FrozenDict from dascore.utils.misc import ( @@ -75,12 +81,6 @@ cached_method, iterate, ) -from dascore.utils.models import ( - ArrayLike, - DascoreBaseModel, - frozen_dict_serializer, - frozen_dict_validator, -) MaybeArray = ArrayLike | np.ndarray | None diff --git a/dascore/core/coords.py b/dascore/core/coords.py index f2a0d7435..a0cf22c17 100644 --- a/dascore/core/coords.py +++ b/dascore/core/coords.py @@ -35,6 +35,11 @@ from dascore.compat import array, is_array from dascore.constants import _AGG_FUNCS, DIM_REDUCE_DOCS, dascore_styles from dascore.exceptions import CoordError, ParameterError +from dascore.models import ( + ArrayLike, + DascoreBaseModel, + UnitQuantity, +) from dascore.units import ( Quantity, Unit, @@ -63,11 +68,6 @@ iterate, sanitize_range_param, ) -from dascore.utils.models import ( - ArrayLike, - DascoreBaseModel, - UnitQuantity, -) from dascore.utils.time import ( dtype_time_like, is_datetime64, diff --git a/dascore/core/inventory.py b/dascore/core/inventory.py index 47fae6b60..ef1de8c7a 100644 --- a/dascore/core/inventory.py +++ b/dascore/core/inventory.py @@ -44,6 +44,14 @@ from dascore.constants import DataCategory, DataType from dascore.exceptions import InvalidInventoryError, ParameterError +from dascore.models import ( + DateTime64, + FiniteFloat, + FrozenDictType, + InventoryModel, + TimeRangedModel, + UnitQuantity, +) from dascore.utils.mapping import FrozenDict from dascore.utils.misc import ( check_code, @@ -51,13 +59,6 @@ optional_import, validate_acquisition_key, ) -from dascore.utils.models import ( - DateTime64, - FrozenDictType, - InventoryModel, - TimeRangedModel, - UnitQuantity, -) CouplingType = Literal[ "conduit", @@ -90,8 +91,6 @@ # The token rule is shared with PatchAttrs.acquisition_key so a code legal # in one is legal in the other. CodeStr = Annotated[str, AfterValidator(check_code)] -# A float which must be finite; nan/inf silently poison downstream math. -FiniteFloat = Annotated[float, Field(allow_inf_nan=False)] # Sensor orientation, in the ranges seismology already uses. Azimuth = Annotated[float, Field(ge=0, lt=360, allow_inf_nan=False)] Dip = Annotated[float, Field(ge=-90, le=90, allow_inf_nan=False)] @@ -129,14 +128,17 @@ def _annotation_value(value): ] -def _type_tag(name: str): +def _object_type_tag(name: str): """ - Return the serialization-only type tag field for a tagged model class. + Return the serialization-only ``object_type`` field of a union member. + + Every model states its class in a json document (see + [dascore.models.registry](`dascore.models.registry`)), but pydantic must + pick a union member's class before an object exists, so these models + declare the tag as a real field and the base class leaves them alone. - The tag drives union dispatch and the authoring format's type - declaration in serialized YAML/JSON. Users never set it: it defaults to - the class name, the Literal annotation rejects any other value, and it - is hidden from repr. + Users never set it: it defaults to the class name, the Literal + annotation rejects any other value, and it is hidden from repr. """ return Field(default=name, repr=False) @@ -254,7 +256,7 @@ def axis_index(self, label: str) -> int: class ExternalResource(InventoryModel): """External resource identified but not otherwise modeled by DASCore.""" - type: Literal["ExternalResource"] = _type_tag("ExternalResource") + object_type: Literal["ExternalResource"] = _object_type_tag("ExternalResource") resource_id: ResourceIdStr uri: str = Field(default="", description="URI or identifier for the resource.") name: str = Field(default="", description="Human-readable resource name.") @@ -270,7 +272,7 @@ class OpticalMeasurement(InventoryModel): datasheet claim is a legitimate record (method="datasheet"). """ - type: Literal["OpticalMeasurement"] = _type_tag("OpticalMeasurement") + object_type: Literal["OpticalMeasurement"] = _object_type_tag("OpticalMeasurement") resource_id: ResourceIdStr name: str = Field(default="", description="Human-readable measurement name.") method: str = Field( @@ -301,7 +303,7 @@ class OpticalMeasurement(InventoryModel): class Interrogator(InventoryModel): """DFOS interrogator unit used for data collection.""" - type: Literal["Interrogator"] = _type_tag("Interrogator") + object_type: Literal["Interrogator"] = _object_type_tag("Interrogator") resource_id: ResourceIdStr name: str = Field(default="", description="Human-readable resource name.") manufacturer: str = Field(default="", description="Manufacturer name.") @@ -315,7 +317,7 @@ class Interrogator(InventoryModel): class Enclosure(InventoryModel): """Physical housing, pipe, duct, conduit, or carrier resource.""" - type: Literal["Enclosure"] = _type_tag("Enclosure") + object_type: Literal["Enclosure"] = _object_type_tag("Enclosure") resource_id: ResourceIdStr name: str = Field(default="", description="Human-readable resource name.") enclosure_type: str = Field( @@ -340,7 +342,7 @@ class Enclosure(InventoryModel): class Cable(InventoryModel): """Physical cable containing one or more fiber segments.""" - type: Literal["Cable"] = _type_tag("Cable") + object_type: Literal["Cable"] = _object_type_tag("Cable") resource_id: ResourceIdStr name: str = Field(default="", description="Human-readable resource name.") manufacturer: str = Field(default="", description="Manufacturer name.") @@ -361,7 +363,7 @@ class Cable(InventoryModel): _Resource: TypeAlias = Annotated[ Interrogator | Cable | Enclosure | ExternalResource | OpticalMeasurement, - Field(discriminator="type"), + Field(discriminator="object_type"), ] @@ -432,7 +434,7 @@ def _check_measurement_pairing(self) -> Self: class FiberSegment(_OpticalComponentBase): """Length of optical fiber within a cable, patch cord, or other run.""" - type: Literal["FiberSegment"] = _type_tag("FiberSegment") + object_type: Literal["FiberSegment"] = _object_type_tag("FiberSegment") container: Cable | str | None = Field( default=None, description="Cable containing this fiber." ) @@ -471,7 +473,7 @@ def attenuation_db_per_km(self) -> float | tuple[float, ...] | None: class Connector(_OpticalComponentBase): """Optical connector in an optical path.""" - type: Literal["Connector"] = _type_tag("Connector") + object_type: Literal["Connector"] = _object_type_tag("Connector") container: Enclosure | str | None = Field( default=None, description="Enclosure housing this connector." ) @@ -481,7 +483,7 @@ class Connector(_OpticalComponentBase): class Splice(_OpticalComponentBase): """Optical splice in an optical path.""" - type: Literal["Splice"] = _type_tag("Splice") + object_type: Literal["Splice"] = _object_type_tag("Splice") container: Enclosure | str | None = Field( default=None, description="Enclosure housing this splice." ) @@ -491,7 +493,7 @@ class Splice(_OpticalComponentBase): class Terminator(_OpticalComponentBase): """Optical path terminator.""" - type: Literal["Terminator"] = _type_tag("Terminator") + object_type: Literal["Terminator"] = _object_type_tag("Terminator") container: Enclosure | str | None = Field( default=None, description="Enclosure housing this terminator." ) @@ -502,7 +504,7 @@ class Terminator(_OpticalComponentBase): OpticalComponent: TypeAlias = Annotated[ FiberSegment | Connector | Splice | Terminator, - Field(discriminator="type"), + Field(discriminator="object_type"), ] @@ -1687,7 +1689,7 @@ class InventoryNames(NamedTuple): # Fields which name or tag a record rather than describe it: the type # discriminator and resource id a shareable record carries, and the codes # locating an acquisition (a patch's acquisition_key already states them). -_IDENTITY_FIELDS = frozenset({"type", "resource_id", "code", "location_code"}) +_IDENTITY_FIELDS = frozenset({"object_type", "resource_id", "code", "location_code"}) # What a distance-ranged track carries to place itself, which is its extent # rather than a value the channels inside it take. diff --git a/dascore/core/inventory_loader.py b/dascore/core/inventory_loader.py index 9c023f0d5..33b26c705 100644 --- a/dascore/core/inventory_loader.py +++ b/dascore/core/inventory_loader.py @@ -13,7 +13,7 @@ skipped, so a directory cannot load as an entity silently missing the tracks its own files state. -The contract, in one line: **file declares type, container agrees, name +The contract, in one line: **file declares object_type, container agrees, name implies identity, envelope implies version.** Every object file states what it is, its container checks that statement rather than supplying it, its name decides which entity it is, and the top-level ``inventory.yaml`` @@ -56,8 +56,9 @@ InvalidInventoryError, MissingOptionalDependencyError, ) +from dascore.models import InventoryModel, TimeRangedModel +from dascore.models.registry import TAG_FIELD from dascore.utils.misc import check_code, optional_import -from dascore.utils.models import InventoryModel, TimeRangedModel from dascore.utils.time import to_datetime64 # One data model stands behind all three spellings, so they are accepted @@ -229,7 +230,7 @@ def _declared_type(path: Path) -> str | None: data = _read_object(path) except (InvalidInventoryError, MissingOptionalDependencyError): return None - declared = data.get("type") + declared = data.get(TAG_FIELD) return declared if isinstance(declared, str) else None @@ -317,27 +318,26 @@ def _pick_model(data: dict, container: _Container, source: Path): The container never supplies the type: a file which does not say what it is has not participated in the format, and one which says something its container cannot hold is misfiled rather than reinterpreted. + + The tag is left in the data. Every model reads its own, so the file's + statement is checked a second time by the object it builds. """ - declared = data.get("type") + declared = data.get(TAG_FIELD) legal = tuple(x.__name__ for x in container.models) # Not merely absent: a type which is not a name at all -- a list, a # mapping -- names no model either, and must not reach the lookups below. if not isinstance(declared, str): msg = ( - f"{_quote(source)} declares no type. Every object file states " - f"what it is, e.g. 'type: {legal[0]}'." + f"{_quote(source)} declares no {TAG_FIELD}. Every object file " + f"states what it is, e.g. '{TAG_FIELD}: {legal[0]}'." ) raise InvalidInventoryError(msg) for model in container.models: if model.__name__ == declared: - # The tag is a real, discriminating field only on the models - # which share a union; everywhere else it belongs to the format. - if "type" not in model.model_fields: - data.pop("type") return model known = "an inventory model" if declared in _model_names() else "unknown" msg = ( - f"{_quote(source)} declares type {declared!r} ({known}), which " + f"{_quote(source)} declares {TAG_FIELD} {declared!r} ({known}), which " f"{source.parent.name} cannot hold. Expected one of {legal}." ) raise InvalidInventoryError(msg) @@ -793,10 +793,10 @@ def _load_envelope(root: Path) -> dict[str, Any] | None: raise InvalidInventoryError(msg) source = found[0] data = _read_object(source) - if (declared := data.pop("type", None)) != Inventory.__name__: + if (declared := data.get(TAG_FIELD)) != Inventory.__name__: msg = ( - f"{_quote(source)} declares type {declared!r}; the envelope " - f"declares 'type: {Inventory.__name__}'." + f"{_quote(source)} declares {TAG_FIELD} {declared!r}; the envelope " + f"declares '{TAG_FIELD}: {Inventory.__name__}'." ) raise InvalidInventoryError(msg) _refuse_supplied(data, _ENVELOPE_COLLECTIONS, source, "the envelope") @@ -837,7 +837,8 @@ def check(path: Path): elif _object_suffix(path) is not None: if (declared := _declared_type(path)) in known: msg = ( - f"{path.relative_to(root)} declares type {declared!r} but " + f"{path.relative_to(root)} declares {TAG_FIELD} " + f"{declared!r} but " "nothing contains it. An inventory object lives in one of " f"{tuple(_CONTAINERS)}, named for the entity it is." ) diff --git a/dascore/core/patch.py b/dascore/core/patch.py index 1ccc8f3ca..81253d4d2 100644 --- a/dascore/core/patch.py +++ b/dascore/core/patch.py @@ -18,6 +18,7 @@ from dascore.core.attrs import PatchAttrs from dascore.core.coordmanager import CoordManager, get_coord_manager from dascore.core.summary import PatchSummary +from dascore.models import ArrayLike from dascore.utils.array import ( PatchUFunc, apply_ufunc, @@ -25,7 +26,6 @@ patch_array_ufunc, ) from dascore.utils.display import array_to_text, attrs_to_text, get_dascore_text -from dascore.utils.models import ArrayLike from dascore.utils.namespace import NamespaceOwner from dascore.utils.patch import check_patch_attrs, check_patch_coords, get_patch_names from dascore.utils.time import to_float diff --git a/dascore/core/summary.py b/dascore/core/summary.py index ba3ab5dec..a49c62cbe 100644 --- a/dascore/core/summary.py +++ b/dascore/core/summary.py @@ -11,13 +11,13 @@ import numpy as np import pandas as pd -from pydantic import ConfigDict, Field, model_validator +from pydantic import ConfigDict, Field, SerializeAsAny, model_validator import dascore as dc from dascore.constants import path_types from dascore.core.attrs import PatchAttrs from dascore.core.coords import CoordSummary -from dascore.utils.models import DascoreBaseModel +from dascore.models import DascoreBaseModel from dascore.utils.paths import coerce_to_upath, is_pathlike @@ -202,7 +202,7 @@ class PatchSummary(DascoreBaseModel): model_config = ConfigDict(title="Patch Summary", extra="ignore", frozen=True) - attrs: PatchAttrs = Field(default_factory=PatchAttrs) + attrs: SerializeAsAny[PatchAttrs] = Field(default_factory=PatchAttrs) coords: dict[str, CoordSummary] = Field(default_factory=dict) dims: tuple[str, ...] = () shape: tuple[int, ...] = () diff --git a/dascore/exceptions.py b/dascore/exceptions.py index c158b0f3a..df9f90301 100644 --- a/dascore/exceptions.py +++ b/dascore/exceptions.py @@ -171,3 +171,7 @@ class RemoteCacheError(IOError, DASCoreError): class InvalidInventoryError(ValueError, DASCoreError): """Raised when inventory metadata violates the DASDAE inventory model.""" + + +class InvalidModelTagError(ValueError, DASCoreError): + """Raised when a serialized document names its model class illegally.""" diff --git a/dascore/io/ai4eps/core.py b/dascore/io/ai4eps/core.py index b41ab0cd6..0192fecc8 100644 --- a/dascore/io/ai4eps/core.py +++ b/dascore/io/ai4eps/core.py @@ -9,8 +9,8 @@ import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload +from dascore.models import DateTime64, OptionalFiniteFloat from dascore.utils.hdf5 import H5Reader -from dascore.utils.models import DateTime64 from .utils import _get_attrs_dict, _get_coords, _get_patches, _is_ai4eps @@ -19,12 +19,12 @@ class AI4EPSPatchAttrs(dc.PatchAttrs): """Patch attributes for AI4EPS event files.""" event_id: str = "" - event_time: DateTime64 = np.datetime64("NaT") - magnitude: float = np.nan + event_time: DateTime64 = np.datetime64("NaT", "ns") + magnitude: OptionalFiniteFloat = None magnitude_type: str = "" - event_latitude: float = np.nan - event_longitude: float = np.nan - event_depth_km: float = np.nan + event_latitude: OptionalFiniteFloat = None + event_longitude: OptionalFiniteFloat = None + event_depth_km: OptionalFiniteFloat = None class AI4EPSV1(FiberIO): diff --git a/dascore/io/ap_sensing/core.py b/dascore/io/ap_sensing/core.py index 09498c239..e33c832fd 100644 --- a/dascore/io/ap_sensing/core.py +++ b/dascore/io/ap_sensing/core.py @@ -6,11 +6,10 @@ from typing import Literal -import numpy as np - import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload +from dascore.models import OptionalFiniteFloat from dascore.utils.hdf5 import H5Reader from .utils import _get_attrs_dict, _get_coords, _get_patches, _get_version_string @@ -19,8 +18,8 @@ class APSensingPatchAttrs(dc.PatchAttrs): """Patch Attributes for AP sensing.""" - gauge_length: float = np.nan - radians_to_nano_strain: float = np.nan + gauge_length: OptionalFiniteFloat = None + radians_to_nano_strain: OptionalFiniteFloat = None class APSensingV10(FiberIO): diff --git a/dascore/io/core.py b/dascore/io/core.py index 71d530e64..c53cf9075 100644 --- a/dascore/io/core.py +++ b/dascore/io/core.py @@ -77,7 +77,7 @@ warn_or_raise, ) from dascore.utils.paths import coerce_to_local_path, coerce_to_upath, is_local_path -from dascore.utils.plugins import get_entry_point_loaders +from dascore.utils.plugins import FIBER_IO_GROUP, get_entry_point_loaders from dascore.utils.progress import track from dascore.utils.remote_io import ( get_remote_cache_scope, @@ -490,7 +490,7 @@ def _eps(self): Get the unloaded entry points registered to this domain into a dict of {name: ep}. """ - return pd.Series(get_entry_point_loaders("dascore.fiber_io")) + return pd.Series(get_entry_point_loaders(FIBER_IO_GROUP)) @cached_property @_locked("_lock") @@ -940,7 +940,7 @@ class FiberIO: # True when a single resource can hold more than one patch. multi_patch_write: bool = False - manager = _FiberIOManager("dascore.fiber_io") + manager = _FiberIOManager(FIBER_IO_GROUP) # A dict of methods which should implement automatic type casting. # and the index of the parameter to type cast. diff --git a/dascore/io/dasdae/utils.py b/dascore/io/dasdae/utils.py index bf7673ef6..c6aea96b1 100644 --- a/dascore/io/dasdae/utils.py +++ b/dascore/io/dasdae/utils.py @@ -18,6 +18,7 @@ from dascore.io.core import make_scan_payload from dascore.io.dasdae._compat import strip_legacy_coord_fields, translate_legacy_attrs from dascore.io.utils import get_exact_coord +from dascore.models.registry import get_model_tag, resolve_tagged_model from dascore.utils.array import ( convert_bytes_to_strings, convert_strings_to_bytes, @@ -34,6 +35,10 @@ # Root marker set on files whose patch attr namespace holds only true attrs. # Files without it may mix flat coord metadata into attrs (see _compat). _SEPARATE_ATTRS_KEY = "__attrs_coords_separate__" +# Names the attrs class a patch group holds. A sibling of the attr +# namespace rather than a member of it, since attrs allow extras and a +# patch may carry one spelled like this key. +_ATTRS_CLASS_KEY = "__attrs_class__" # --- Functions for writing DASDAE format @@ -90,6 +95,12 @@ def _save_attrs_and_dims(patch, patch_group): patch_group.attrs[f"{_ATTR_PREFIX}{i}"] = encoded if attr_type is not None: patch_group.attrs[f"{_ATTR_TYPE_PREFIX}{i}"] = attr_type + # Values are dumped one at a time rather than as one document, so the + # class is recorded beside them rather than injected into them. A class + # which cannot be named (see get_model_tag) is simply not named, which + # reads back the way a file written before this did. + if (tag := get_model_tag(type(patch.attrs))) is not None: + patch_group.attrs[_ATTRS_CLASS_KEY] = tag patch_group.attrs["_dims"] = ",".join(patch.dims) @@ -170,6 +181,19 @@ def _get_attrs(patch_group): return out +def _get_attrs_class(patch_group) -> type[PatchAttrs]: + """ + Return the attrs class a patch group names, or the base class. + + A file written before the class was recorded names nothing, and one + written by a format which is no longer installed names something + unresolvable; both read as plain attrs, which is what such a file + always used to give. + """ + tag = unbyte(patch_group.attrs.get(_ATTRS_CLASS_KEY, None)) + return resolve_tagged_model(tag or None, default=PatchAttrs) + + def _read_array(table_array): """Read an array into numpy.""" data = table_array[:] @@ -299,7 +323,7 @@ def _read_patch(patch_group, legacy: bool = True, **kwargs): coords = _get_coords(patch_group, dims, {}) attr_info = attrs attr_info["_source_patch_id"] = patch_group.name.rsplit("/", maxsplit=1)[-1] - attrs = PatchAttrs.from_dict(attr_info) + attrs = _get_attrs_class(patch_group).from_dict(attr_info) # Note, previously this was wrapped with try, except (Index, KeyError) # and the data = np.array(None) in except block. Not sure, why, removed # try except. @@ -361,7 +385,7 @@ def _get_scan_payload_from_group(group, legacy: bool = True, snap=True): dtype = str(data_node.dtype) if data_node is not None else "" shape = tuple(data_node.shape) if data_node is not None else () return make_scan_payload( - attrs=PatchAttrs.from_dict(attr_info), + attrs=_get_attrs_class(group).from_dict(attr_info), coords=coords, dims=dims, shape=shape, diff --git a/dascore/io/febus/core.py b/dascore/io/febus/core.py index fac9dd08e..590dde056 100644 --- a/dascore/io/febus/core.py +++ b/dascore/io/febus/core.py @@ -7,8 +7,6 @@ import warnings from typing import Literal -import numpy as np - import dascore as dc from dascore.constants import ( float_select_type, @@ -18,9 +16,9 @@ ) from dascore.io import FiberIO, ScanPayload from dascore.io.core import make_scan_payload +from dascore.models import OptionalFiniteFloat, UTF8Str from dascore.utils.hdf5 import H5Reader from dascore.utils.io import TextReader -from dascore.utils.models import UTF8Str from .a1utils import ( _get_febus_version_str, @@ -61,8 +59,8 @@ class FebusPatchAttrs(dc.PatchAttrs): The zone designations """ - gauge_length: float = np.nan - pulse_length: float = np.nan + gauge_length: OptionalFiniteFloat = None + pulse_length: OptionalFiniteFloat = None group: str = "" source: str = "" diff --git a/dascore/io/gdr/core.py b/dascore/io/gdr/core.py index a55e34a76..b1bfd2842 100644 --- a/dascore/io/gdr/core.py +++ b/dascore/io/gdr/core.py @@ -14,13 +14,14 @@ from dascore.io import FiberIO, ScanPayload, make_scan_payload from dascore.io.gdr.utils_das import _get_attrs_coords_and_data, _get_version from dascore.io.utils import build_patches +from dascore.models import OptionalFiniteFloat from dascore.utils.hdf5 import H5Reader class GDRPatchAttrs(dc.PatchAttrs): """Patch attrs for GDR files.""" - gauge_length: float + gauge_length: OptionalFiniteFloat project_number: str = "" diff --git a/dascore/io/neubrex/core.py b/dascore/io/neubrex/core.py index 4aa9bcb89..5a6307cd8 100644 --- a/dascore/io/neubrex/core.py +++ b/dascore/io/neubrex/core.py @@ -13,6 +13,7 @@ import dascore.io.neubrex.utils_rfs as rfs_utils from dascore.io import FiberIO, ScanPayload, make_scan_payload from dascore.io.utils import build_patches +from dascore.models import OptionalFiniteFloat from dascore.utils.hdf5 import H5Reader @@ -29,10 +30,10 @@ class NeubrexRFSPatchAttrs(dc.PatchAttrs): class NeubrexDASPatchAttrs(dc.PatchAttrs): """Patch attrs for Neubrex DAS Format files.""" - gauge_length: float = 0 - index_of_reflection: float = 1.46 + gauge_length: OptionalFiniteFloat = 0 + index_of_reflection: OptionalFiniteFloat = 1.46 triggered_time: np.datetime64 | None = None - phase_to_strain: float | None = None + phase_to_strain: OptionalFiniteFloat = None distance_decimation_filter: int = 0 time_decimation_filter: int = 0 diff --git a/dascore/io/odh4/core.py b/dascore/io/odh4/core.py index b76befb5e..781abb1ef 100644 --- a/dascore/io/odh4/core.py +++ b/dascore/io/odh4/core.py @@ -4,11 +4,10 @@ from typing import Literal -import numpy as np - import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload +from dascore.models import OptionalFiniteFloat from dascore.utils.hdf5 import H5Reader from .utils import _get_attrs_dict, _get_coords, _get_patches, _is_odh4, _read_attrs @@ -17,8 +16,8 @@ class ODH4PatchAttrs(dc.PatchAttrs): """Patch attributes for ODH4 files.""" - gauge_length: float = np.nan - scale_factor_to_strain: float = np.nan + gauge_length: OptionalFiniteFloat = None + scale_factor_to_strain: OptionalFiniteFloat = None class ODH4V1(FiberIO): diff --git a/dascore/io/optodas/core.py b/dascore/io/optodas/core.py index f95e323ea..1ce7fc0a5 100644 --- a/dascore/io/optodas/core.py +++ b/dascore/io/optodas/core.py @@ -4,13 +4,11 @@ from typing import Literal -import numpy as np - import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload +from dascore.models import OptionalFiniteFloat, UTF8Str from dascore.utils.hdf5 import H5Reader -from dascore.utils.models import UTF8Str from .utils import _get_opto_das_attrs, _get_opto_das_version_str, _read_opto_das @@ -18,7 +16,7 @@ class OptoDASPatchAttrs(dc.PatchAttrs): """Patch attrs for OptoDAS.""" - gauge_length: float = np.nan + gauge_length: OptionalFiniteFloat = None schema_version: UTF8Str = "" diff --git a/dascore/io/prodml/utils.py b/dascore/io/prodml/utils.py index 4625523a9..1b12b7e27 100644 --- a/dascore/io/prodml/utils.py +++ b/dascore/io/prodml/utils.py @@ -15,11 +15,11 @@ from dascore.core.coords import get_coord from dascore.exceptions import InvalidSpoolError, PatchError from dascore.io.utils import convert_attr_units, get_exact_coord +from dascore.models import OptionalFiniteFloat, UTF8Str from dascore.units import get_quantity_str from dascore.utils.hdf5 import encode_h5_strings from dascore.utils.io import _normalize_source_patch_ids from dascore.utils.misc import iterate, maybe_get_items, register_func, unbyte -from dascore.utils.models import UTF8Str # --- Getting format/version @@ -77,8 +77,8 @@ class ProdMLRawPatchAttrs(dc.PatchAttrs): """Patch attrs for raw data contained in ProdML.""" - pulse_width: float = np.nan - gauge_length: float = np.nan + pulse_width: OptionalFiniteFloat = None + gauge_length: OptionalFiniteFloat = None schema_version: UTF8Str = "" @@ -86,13 +86,13 @@ class ProdMLFbePatchAttrs(ProdMLRawPatchAttrs): """Patch attrs for fbe (frequency band extracted) data in Prodml.""" raw_reference: UTF8Str = "" - transform_size: float = np.nan + transform_size: OptionalFiniteFloat = None transform_type: UTF8Str = "" window_size: int | None = None window_function: UTF8Str = "" window_overlap: int | None = None - start_frequency: float = 0 - end_frequency: float = np.inf + start_frequency: OptionalFiniteFloat = 0 + end_frequency: OptionalFiniteFloat = None @dataclass diff --git a/dascore/io/silixah5/core.py b/dascore/io/silixah5/core.py index 8fefe4f87..2efcdad06 100644 --- a/dascore/io/silixah5/core.py +++ b/dascore/io/silixah5/core.py @@ -6,11 +6,10 @@ from typing import Literal -import numpy as np - import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload +from dascore.models import OptionalFiniteFloat from dascore.utils.hdf5 import H5Reader from .utils import ( @@ -26,8 +25,8 @@ class SilixaPatchAttrs(dc.PatchAttrs): """Patch Attributes for Silixa hdf5 format.""" - gauge_length: float = np.nan - pulse_width: float = np.nan + gauge_length: OptionalFiniteFloat = None + pulse_width: OptionalFiniteFloat = None class SilixaH5V1(FiberIO): diff --git a/dascore/io/sintela/core.py b/dascore/io/sintela/core.py index ca49844bc..ad3edc032 100644 --- a/dascore/io/sintela/core.py +++ b/dascore/io/sintela/core.py @@ -11,6 +11,7 @@ import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload +from dascore.models import OptionalFiniteFloat from dascore.utils.io import BinaryReader, LocalBinaryReader from .protobuf_utils import get_supported_family_tag, read_payload, scan_payload @@ -26,7 +27,7 @@ class SintelaPatchAttrs(dc.PatchAttrs): """Patch Attributes for Sintela binary format.""" - gauge_length: float = np.nan + gauge_length: OptionalFiniteFloat = None class SintelaBinaryV3(FiberIO): diff --git a/dascore/io/sintela/protobuf_utils.py b/dascore/io/sintela/protobuf_utils.py index fdcaf67ce..5bc31f027 100644 --- a/dascore/io/sintela/protobuf_utils.py +++ b/dascore/io/sintela/protobuf_utils.py @@ -48,7 +48,7 @@ from typing import Any import numpy as np -from pydantic import ValidationError +from pydantic import BaseModel, ConfigDict, ValidationError import dascore as dc from dascore.core.attrs import PatchAttrs @@ -56,8 +56,8 @@ from dascore.core.coords import get_coord from dascore.exceptions import InvalidFiberFileError from dascore.io.core import ScanPayload, make_scan_payload +from dascore.models import OptionalFiniteFloat, PositiveFiniteFloat, PositiveInt from dascore.utils.misc import optional_import, suppress_warnings -from dascore.utils.models import DascoreBaseModel, PositiveFiniteFloat, PositiveInt PBUF_MAGIC = 0x46554250 META_TAG = "META" @@ -143,7 +143,7 @@ def _get_fft_data_type(has_complex: bool) -> dict[str, str]: class SintelaProtobufAttrs(PatchAttrs): """Patch attributes for Sintela protobuf recordings.""" - gauge_length: float = np.nan + gauge_length: OptionalFiniteFloat = None packet_type: str = "" recorder_namespace: str = "" metadata_recording_time: np.datetime64 | None = None @@ -153,14 +153,27 @@ class SintelaProtobufAttrs(PatchAttrs): demod_data_type: str = "" -class EnvelopeRecord(DascoreBaseModel): +class _ProtobufModel(BaseModel): + """ + Base for this module's parsing models. + + Plain pydantic: these validate values on the way out of a protobuf + payload and are never serialized. Subclassing DascoreBaseModel would + only claim each a tag in the model registry, naming them in documents + they never appear in. + """ + + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + +class EnvelopeRecord(_ProtobufModel): """The envelope information for one MTLV record.""" tag: str payload: bytes -class ParsedMeta(DascoreBaseModel): +class ParsedMeta(_ProtobufModel): """Selected metadata fields promoted from META packets.""" recorder_namespace: str = "" @@ -823,7 +836,7 @@ def _decode_family(parsed: list[tuple[str, Any]], meta: ParsedMeta): return family_cls.from_parsed(parsed, meta).decode(parsed) -class _PacketHeaderFields(DascoreBaseModel): +class _PacketHeaderFields(_ProtobufModel): """ Validated per-packet header fields shared by all families. @@ -871,7 +884,7 @@ def _reduce_common(fields: list[_PacketHeaderFields]): ) -class _PacketMetadata(DascoreBaseModel): +class _PacketMetadata(_ProtobufModel): """ Base for a validated, decodable Sintela packet family. diff --git a/dascore/io/sr4731/utils.py b/dascore/io/sr4731/utils.py index 85e6c38e9..6d793a251 100644 --- a/dascore/io/sr4731/utils.py +++ b/dascore/io/sr4731/utils.py @@ -35,6 +35,7 @@ from dascore.exceptions import InvalidFiberFileError from dascore.io.core import ScanPayload, make_scan_payload from dascore.io.utils import build_patches +from dascore.models import OptionalFiniteFloat DIMS = ("time", "distance") REQUIRED_BLOCKS = frozenset( @@ -56,10 +57,10 @@ class Block: class SR4731PatchAttrs(PatchAttrs): """Patch attributes for supported SR-4731 SOR files.""" - wavelength_nm: float = np.nan - acquisition_range_m: float = np.nan - sample_spacing_usec: float = np.nan - refractive_index: float = np.nan + wavelength_nm: OptionalFiniteFloat = None + acquisition_range_m: OptionalFiniteFloat = None + sample_spacing_usec: OptionalFiniteFloat = None + refractive_index: OptionalFiniteFloat = None trace_count: int = 0 sample_scale: int = 0 diff --git a/dascore/io/utils.py b/dascore/io/utils.py index 7ef41de61..b647633e6 100644 --- a/dascore/io/utils.py +++ b/dascore/io/utils.py @@ -13,9 +13,9 @@ from dascore.core.coordmanager import CoordManager from dascore.core.coords import BaseCoord, CoordSegmented, get_coord from dascore.exceptions import CoordError, UnitError +from dascore.models import ArrayLike from dascore.units import convert_units, get_quantity_str from dascore.utils.misc import unbyte -from dascore.utils.models import ArrayLike # Stored coordinate arrays often carry sub-step jitter (e.g. GPS-stamped DAS # time). ``CoordSegmented.from_array`` treats every isolated sampling change as diff --git a/dascore/io/xml_binary/core.py b/dascore/io/xml_binary/core.py index 81c7c00b8..301ee2ede 100644 --- a/dascore/io/xml_binary/core.py +++ b/dascore/io/xml_binary/core.py @@ -5,12 +5,11 @@ from typing import Literal from xml.etree.ElementTree import ParseError -import numpy as np from pydantic import ValidationError import dascore as dc from dascore.io import FiberIO, ScanPayload -from dascore.utils.models import UTF8Str +from dascore.models import OptionalFiniteFloat, UTF8Str from dascore.utils.paths import coerce_to_upath from .utils import _load_patches, _paths_to_scan_patches, _read_xml_metadata @@ -19,8 +18,8 @@ class BinaryPatchAttrs(dc.PatchAttrs): """Patch attrs for Binary.""" - pulse_width: float = np.nan - gauge_length: float = np.nan + pulse_width: OptionalFiniteFloat = None + gauge_length: OptionalFiniteFloat = None zone_name: UTF8Str = "" diff --git a/dascore/io/xml_binary/utils.py b/dascore/io/xml_binary/utils.py index b56d1ea11..23fb8a21b 100644 --- a/dascore/io/xml_binary/utils.py +++ b/dascore/io/xml_binary/utils.py @@ -8,7 +8,7 @@ import numpy as np import pandas as pd -from pydantic import ConfigDict +from pydantic import BaseModel, ConfigDict from pydantic.alias_generators import to_pascal import dascore as dc @@ -16,8 +16,8 @@ from dascore.core import get_coord, get_coord_manager from dascore.io import ScanPayload from dascore.io.core import make_scan_payload +from dascore.models import DateTime64 from dascore.utils.misc import iterate -from dascore.utils.models import BaseModel, DateTime64 from dascore.utils.pd import adjust_segments, filter_df from dascore.utils.remote_io import ensure_local_file from dascore.utils.time import to_float diff --git a/dascore/models/__init__.py b/dascore/models/__init__.py new file mode 100644 index 000000000..977bc613a --- /dev/null +++ b/dascore/models/__init__.py @@ -0,0 +1,57 @@ +""" +DASCore's model layer: base classes and the annotated types they are built from. + +Anything which inherits from [DascoreBaseModel](`dascore.models.base.DascoreBaseModel`) +can appear in a DASCore document. Models used only to shuttle values inside one +module are plain pydantic models instead. +""" + +from __future__ import annotations + +from dascore.models.base import ( + DascoreBaseModel, + InventoryModel, + TimeRangedModel, + sensible_model_equals, + sensible_model_hash, + values_equal, +) +from dascore.models.types import ( + ArrayLike, + CommaSeparatedStr, + DateTime64, + DTypeLike, + FiniteFloat, + FrozenDictType, + OptionalFiniteFloat, + PositiveFiniteFloat, + PositiveInt, + TimeDelta64, + UnitQuantity, + UTF8Str, + frozen_dict_serializer, + frozen_dict_validator, +) + +__all__ = [ + "ArrayLike", + "CommaSeparatedStr", + "DTypeLike", + "DascoreBaseModel", + "DateTime64", + "FiniteFloat", + "FrozenDictType", + "InventoryModel", + "OptionalFiniteFloat", + "PositiveFiniteFloat", + "PositiveInt", + "TimeDelta64", + "TimeRangedModel", + "UTF8Str", + "UnitQuantity", + "frozen_dict_serializer", + "frozen_dict_validator", + "sensible_model_equals", + "sensible_model_hash", + "values_equal", +] diff --git a/dascore/models/base.py b/dascore/models/base.py new file mode 100644 index 000000000..972ddcec1 --- /dev/null +++ b/dascore/models/base.py @@ -0,0 +1,294 @@ +"""Base classes for DASCore's pydantic models.""" + +from __future__ import annotations + +from collections.abc import Mapping +from functools import cached_property +from typing import Any + +import numpy as np +import pandas as pd +from pydantic import ( + BaseModel, + ConfigDict, + Field, + SerializationInfo, + SerializerFunctionWrapHandler, + model_serializer, + model_validator, +) +from typing_extensions import Self + +from dascore.compat import is_array_like +from dascore.exceptions import InvalidInventoryError +from dascore.models.registry import ( + TAG_FIELD, + _lookup, + check_tag_matches, + get_model_tag, + register_model, +) +from dascore.models.types import DateTime64, FrozenDictType +from dascore.utils.misc import _all_null, all_close +from dascore.utils.time import to_datetime64 + + +def sensible_model_equals(self: BaseModel | Mapping, other: object) -> bool: + """Custom equality to not compare private attrs and handle numpy arrays.""" + d1 = self.model_dump() if isinstance(self, BaseModel) else self + if isinstance(other, BaseModel): + d2 = other.model_dump() + elif isinstance(other, Mapping): + d2 = other + else: # nothing else can carry the same fields + return NotImplemented + if not set(d1) == set(d2): # different keys, not equal + return False + for name in set(x for x in d1 if not x.startswith("_")): + # skip any private attributes. + if not values_equal(d1[name], d2[name]): + return False + return True + + +def values_equal(val1, val2) -> bool: + """Recursively compare dumped values; nulls are equal only to nulls.""" + if is_array_like(val1) or is_array_like(val2): + arr1, arr2 = np.asarray(val1), np.asarray(val2) + if arr1.shape != arr2.shape: + return False + if not np.array_equal(pd.isnull(arr1), pd.isnull(arr2)): + return False + return bool(all_close(arr1, arr2)) + if isinstance(val1, Mapping) and isinstance(val2, Mapping): + if set(val1) != set(val2): + return False + return all(values_equal(val1[key], val2[key]) for key in val1) + if isinstance(val1, list | tuple) and isinstance(val2, list | tuple): + if len(val1) != len(val2): + return False + return all(values_equal(v1, v2) for v1, v2 in zip(val1, val2)) + return bool(val1 == val2 or (_all_null(val1) and _all_null(val2))) + + +def _hash_key(value): + """Map a value onto one that hashes the way values_equal compares.""" + if value is None or isinstance(value, str | int): + return value + # Nulls count as equal to one another above, but every nan and NaT is a + # fresh object and both hash by identity, so they collapse to one key. + if isinstance(value, float): + return None if value != value else value + if isinstance(value, np.datetime64 | np.timedelta64): + return None if np.isnat(value) else value + # Mappings are compared without regard to order. + if isinstance(value, Mapping): + return frozenset((k, _hash_key(v)) for k, v in value.items()) + if isinstance(value, tuple): + return tuple(_hash_key(v) for v in value) + return value + + +def sensible_model_hash(self: BaseModel) -> int: + """Hash a model on its fields, agreeing with sensible_model_equals.""" + # Keyed by name and unordered, because equality compares the field names + # it finds rather than the order they were declared in. + fields = type(self).model_fields + return hash(frozenset((x, _hash_key(getattr(self, x))) for x in fields)) + + +class DascoreBaseModel(BaseModel): + """A base model with sensible configurations.""" + + _cache = {} + + model_config = ConfigDict( + extra="ignore", # TODO: change to raise, then let subclass overwrite + validate_assignment=True, + ignored_types=(cached_property,), + frozen=True, + validate_default=True, + arbitrary_types_allowed=True, + ) + + def __init_subclass__(cls, **kwargs): + """Register every model so a document can name it.""" + super().__init_subclass__(**kwargs) + register_model(cls) + + @model_serializer(mode="wrap") + def _write_object_type( + self, handler: SerializerFunctionWrapHandler, info: SerializationInfo + ) -> Any: + """ + Name this class in the document, in json-mode dumps only. + + A python-mode dump is not a document: it is what equality compares, + what ``new`` reconstructs from and what the index ingests, none of + which want a key that is not a field. + + Gated inside the serializer rather than with ``when_used="json"``, + which reads better and silently breaks ``include``/``exclude``: + pydantic skips the whole wrapper in python mode, and with it the + field filtering the handler would have applied. + """ + out = handler(self) + if info.mode != "json" or TAG_FIELD in out: # a union member's own + return out + # Its field is defaulted, so exclude_defaults drops it and leaves a + # document the union cannot dispatch on. Put back what the model + # says rather than what the registry calls the class: a subclass + # out of tree has a different tag, and the Literal would refuse it. + declared = getattr(self, TAG_FIELD, None) + if (tag := declared or get_model_tag(type(self))) is not None: + out[TAG_FIELD] = tag + return out + + @model_validator(mode="before") + @classmethod + def _read_object_type(cls, data: Any) -> Any: + """ + Consume a document's class tag, refusing one which names another class. + + The tag is never required: a document dispatches on it before it + gets here, and a hand-written object or a nested one may simply not + state it. What it may not do is disagree. + + A value which names no known class is left alone rather than + consumed. ``PatchAttrs`` keeps extra fields, so this key may be a + reader's own metadata, and eating it would lose that silently; a + tag DASCore wrote always resolves. + """ + if not isinstance(data, Mapping) or TAG_FIELD not in data: + return data + # A union member declares the tag as a real field and validates it + # against its own Literal, which is stricter than this. + if TAG_FIELD in cls.model_fields: + return data + tag = data[TAG_FIELD] + if (declared := _lookup(tag)) is None: + return data + check_tag_matches(cls, declared, tag) + out = dict(data) + out.pop(TAG_FIELD) + return out + + def new(self, **kwargs) -> Self: + """Create new instance with some attributed updated.""" + out = self.model_dump(exclude_unset=True) + out.update(kwargs) + return self.__class__(**out) + + @classmethod + def get_summary_df(cls): + """Get dataframe of attributes and descriptions for display.""" + fields = cls.model_fields + names_desc = { + i: v.description + for i, v in fields.items() + if getattr(v, "description", False) + } + out = pd.Series(names_desc).to_frame(name="description") + out.index.name = "attribute" + return out + + __eq__ = sensible_model_equals + # Defined together: pydantic would otherwise derive a hash straight from + # the field values, which disagrees with how __eq__ treats nulls. + __hash__ = sensible_model_hash + + +class InventoryModel(DascoreBaseModel): + """ + Base class for immutable DASDAE inventory objects. + + Every inventory object carries two uniform attachment points for + information the model does not otherwise represent: ``description`` + (free prose for humans, matching StationXML's Description element) + and ``extra_fields`` (typed key-values, e.g. for round-tripping + unmodeled metadata from external formats). + + Every field is immutable: collections are tuples and mappings are + frozen, so instances are safe to hold by reference. They hash on their + field values whenever those values are themselves hashable. + """ + + description: str = Field(default="", description="Free-text description.") + extra_fields: FrozenDictType[str, str | int | float | bool] = Field( + default_factory=dict, + description="Extra metadata not represented by standardized fields.", + ) + + model_config = ConfigDict( + extra="forbid", + frozen=True, + validate_assignment=True, + validate_default=True, + arbitrary_types_allowed=True, + ) + + def new(self, **kwargs) -> Self: + """ + Create a new instance with some attributes updated. + + Dumps all fields (not just the set ones) so union discriminators and + validator-normalized fields survive reconstruction. + """ + out = self.model_dump() + out.update(kwargs) + return self.__class__(**out) + + +class TimeRangedModel(InventoryModel): + """Base class for inventory objects with time-validity epochs. + + Validity intervals are half-open, ``[start_time, end_time)``; an unset + (NaT) end time means the epoch is ongoing. All times are UTC. + """ + + start_time: DateTime64 = Field( + default=np.datetime64("NaT", "ns"), + description="Start time for which this metadata item is valid (UTC).", + ) + end_time: DateTime64 = Field( + default=np.datetime64("NaT", "ns"), + description=( + "End time for which this metadata item is valid (UTC); NaT while ongoing." + ), + ) + + @model_validator(mode="after") + def _check_time_order(self): + """A set end time must follow the start time.""" + start, end = self.start_time, self.end_time + if not pd.isnull(start) and not pd.isnull(end) and end <= start: + msg = f"end_time {end} must be after start_time {start}." + raise InvalidInventoryError(msg) + return self + + def is_effective_at(self, time) -> bool: + """Return True if this epoch is valid at the supplied time (half-open).""" + time = to_datetime64(time) + if pd.isnull(time): + return True + start = self.start_time + end = self.end_time + after_start = pd.isnull(start) or start <= time + before_end = pd.isnull(end) or time < end + return bool(after_start and before_end) + + def overlaps(self, other: TimeRangedModel) -> bool: + """ + Return True if two half-open validity intervals overlap. + + Unset (NaT) starts are unbounded past; unset ends are ongoing. + """ + s1, e1, s2, e2 = ( + self.start_time, + self.end_time, + other.start_time, + other.end_time, + ) + first_starts_before = pd.isnull(e2) or pd.isnull(s1) or s1 < e2 + second_starts_before = pd.isnull(e1) or pd.isnull(s2) or s2 < e1 + return bool(first_starts_before and second_starts_before) diff --git a/dascore/models/registry.py b/dascore/models/registry.py new file mode 100644 index 000000000..7ed7d5129 --- /dev/null +++ b/dascore/models/registry.py @@ -0,0 +1,258 @@ +""" +The registry which lets a serialized document name the class it holds. + +A document states its class in an ``object_type`` key holding a registered +name, never an import path. A dotted path would weld stored documents to +today's module layout and make reading one an arbitrary-import surface; +a registered name costs the same to write and has neither property. +""" + +from __future__ import annotations + +import re +import threading +import warnings +from typing import TypeVar, cast + +from dascore.exceptions import InvalidModelTagError +from dascore.utils.plugins import FIBER_IO_GROUP, get_entry_point_loaders + +T = TypeVar("T") + +# The key a document states its class in. +TAG_FIELD = "object_type" + +NAMESPACE_SEP = ":" + +# DASCore's own models are registered bare, so a hand-authored file says +# `object_type: Cable` and a plugin's says `object_type: myplugin:Square`. +DASCORE_NAMESPACE = "dascore" + +# `[namespace:]ClassName[-x.y.z]`. A python class name holds neither ":" +# nor "-", so the three parts can never be read for one another. The +# namespace accepts a capital because package names do (Terra15, PIL). +# Nothing reads a version today; the grammar only keeps room for one, +# where an absent version will mean the earliest. +TAG_PATTERN = re.compile(r"^(?:[A-Za-z_][\w.]*:)?[A-Za-z_]\w*(?:-\d+\.\d+\.\d+)?$") + +_REGISTRY: dict[str, type] = {} + +# Tags two different classes have claimed. Kept rather than resolved to +# either: a file written by the first would otherwise be read as the +# second, silently applying the wrong model to it. +_AMBIGUOUS: dict[str, tuple[str, str]] = {} + +_sweep_lock = threading.Lock() + +# Set once the io plugins have been swept looking for an unresolved tag. +_plugins_swept = False + + +def _derive_namespace(cls: type) -> str: + """Return the namespace a class registers under.""" + # Derived rather than declared: it makes a plugin's models namespaced + # with no ceremony, and leaves no way to squat a bare name. + return cls.__module__.split(".", 1)[0] + + +def get_model_tag(cls: type) -> str | None: + """ + Return the tag which names a model class in a document, if it has one. + + An out-of-tree class is namespaced by the package which declares it; + DASCore's own classes are bare. None means the class cannot be named: + a parametrized generic is spelled ``G[int]``, which no document could + state and this module could not read back. Writing such a tag anyway + would produce a file DASCore refuses to read. + """ + namespace = _derive_namespace(cls) + if namespace == DASCORE_NAMESPACE: + tag = cls.__name__ + else: + tag = f"{namespace}{NAMESPACE_SEP}{cls.__name__}" + return tag if TAG_PATTERN.match(tag) else None + + +def register_model(cls: type) -> None: + """ + Add a model class to the registry under its derived tag. + + Classes declared inside a function are skipped: nothing can resolve a + name which only exists while its enclosing call runs, and two of them + sharing a name is neither a mistake nor resolvable. So are classes + whose derived tag is not a legal one; see `get_model_tag`. + """ + if "" in cls.__qualname__ or (tag := get_model_tag(cls)) is None: + return + existing = _REGISTRY.get(tag) + if existing is not None and _identity(existing) != _identity(cls): + _report_collision(tag, existing, cls) + return + # A module re-imported under the same name replaces its own entry. + _REGISTRY[tag] = cls + + +def _identity(cls: type) -> tuple[str, str]: + """Return what makes a class the same class across a re-import.""" + return (cls.__module__, cls.__qualname__) + + +def _spell(cls: type) -> str: + """Spell a class the way a collision message needs to.""" + return f"{cls.__module__}.{cls.__qualname__}" + + +def _report_collision(tag: str, existing: type, new: type) -> None: + """Complain that two different classes want one tag.""" + msg = ( + f"Two models claim the tag {tag!r}: {_spell(existing)} and " + f"{_spell(new)}. A tag must name one class; rename one of them." + ) + # DASCore's own names are its own to keep unique, and a test pins it. + if _derive_namespace(new) == DASCORE_NAMESPACE: + raise InvalidModelTagError(msg) + # Out of tree the collision may be between two packages a user merely + # installed, which they cannot fix by renaming, so importing them both + # still works. What the tag may not do is quietly resolve to one of + # them: a file written by the first would then be read as the second. + _AMBIGUOUS[tag] = (_spell(existing), _spell(new)) + _REGISTRY.pop(tag, None) + warnings.warn(f"{msg} Documents naming it can no longer be read.", UserWarning) + + +def _sweep_plugin_modules() -> None: + """ + Import the io plugins, defining any models they declare. + + Not `FiberIO.manager.load_plugins()`, which does the same thing and + more: `dascore.io.core` imports `dascore.core.attrs`, which imports + this module, so naming it here at module scope is a cycle -- and a + function-level import is banned by PLC0415. + """ + global _plugins_swept + if _plugins_swept: + return + with _sweep_lock: + # The flag is set after the imports rather than before, so a thread + # arriving mid-sweep waits here rather than reading a registry which + # is still filling and concluding a class is not installed. Sweeping + # again after that wait costs nothing: every module it names is by + # then in sys.modules. + for loader in get_entry_point_loaders(FIBER_IO_GROUP).values(): + try: + loader() + except Exception: + # A plugin which cannot be imported has no models to find. + # It is not reported here: FiberIO warns about the same + # plugin when it loads formats, and failing to resolve one + # tag is not the place to announce a broken install. + continue + _plugins_swept = True + + +def _lookup(tag: object) -> type | None: + """Return the class a tag names, or None if nothing usable names it.""" + if not isinstance(tag, str) or not TAG_PATTERN.match(tag) or tag in _AMBIGUOUS: + return None + if (cls := _REGISTRY.get(tag)) is not None: + return cls + # A format's models only exist once its module is imported, and io + # modules are imported lazily, so an unknown name is worth one sweep. + _sweep_plugin_modules() + return _REGISTRY.get(tag) + + +def resolve_model_tag(tag: str) -> type | None: + """ + Return the class a tag names, or None if nothing registers it. + + Raises if the tag could never name a class -- it is not a legal tag, + or two classes have claimed it -- which is a document this process + cannot read rather than one whose class it merely does not have. + """ + if not isinstance(tag, str) or not TAG_PATTERN.match(tag): + msg = ( + f"{tag!r} is not a legal {TAG_FIELD}. Expected a registered name, " + "optionally namespaced, eg 'Cable' or 'myplugin:Square'." + ) + raise InvalidModelTagError(msg) + if tag in _AMBIGUOUS: + first, second = _AMBIGUOUS[tag] + msg = ( + f"The {TAG_FIELD} {tag!r} names two classes, {first} and " + f"{second}, so which one wrote a document cannot be known." + ) + raise InvalidModelTagError(msg) + return _lookup(tag) + + +def check_tag_matches(cls: type, declared: type, tag: str) -> None: + """ + Refuse a document whose tag names some other class. + + A tag naming a subclass is accepted: such a document holds everything + the class being built declares, which is what a caller asking for the + base class asked for. + """ + if issubclass(declared, cls): + return + msg = ( + f"A document declaring {TAG_FIELD} {tag!r} cannot be read as " + f"{cls.__name__}: {declared.__name__} is not one." + ) + raise InvalidModelTagError(msg) + + +def resolve_tagged_model( + tag: str | None, + default: type[T] | None = None, + source: str | None = None, +) -> type[T]: + """ + Return the model class a tag names. + + Parameters + ---------- + tag + What the document declared, or None if it declared nothing. + default + The class to fall back on when the tag names one which is not + registered, usually because a plugin which wrote it is not + installed. Without one, an unresolved tag raises. + source + Where the document came from, named in errors. Deliberately not in + the warning below, which would otherwise defeat the de-duplication + that keeps a spool of many files from warning once per file. + """ + where = f" in {source}" if source else "" + if tag is None: + if default is None: + msg = ( + f"The document{where} declares no {TAG_FIELD}, and nothing " + "else says which class it holds." + ) + raise InvalidModelTagError(msg) + return default + if (cls := resolve_model_tag(tag)) is not None: + # The registry holds every model, not just the caller's kind, so + # this check is what makes the returned class a `default` at all. + if default is not None and not issubclass(cls, default): + msg = ( + f"The document{where} declares {TAG_FIELD} {tag!r}, which is " + f"not a {default.__name__}." + ) + raise InvalidModelTagError(msg) + return cast("type[T]", cls) + msg = ( + f"Nothing registers the {TAG_FIELD} {tag!r}. It was likely written " + "by a package which is not installed." + ) + if default is None: + raise InvalidModelTagError(f"{msg[:-1]}{where}.") + warnings.warn(f"{msg} Reading it as {default.__name__}.", UserWarning, stacklevel=2) + return default + + +def registered_models() -> dict[str, type]: + """Return the registered classes, keyed by the tag naming each.""" + return dict(_REGISTRY) diff --git a/dascore/models/types.py b/dascore/models/types.py new file mode 100644 index 000000000..833598814 --- /dev/null +++ b/dascore/models/types.py @@ -0,0 +1,125 @@ +"""Annotated types with DASCore's validation and serialization attached.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Annotated, TypeVar + +import numpy as np +from pydantic import ( + AfterValidator, + BeforeValidator, + Field, + PlainSerializer, + PlainValidator, +) + +from dascore.compat import array +from dascore.units import Quantity, get_quantity, get_quantity_str +from dascore.utils.mapping import FrozenDict +from dascore.utils.misc import to_str, unbyte +from dascore.utils.time import to_datetime64, to_timedelta64 + +# --- A list of custom types with appropriate serialization/deserialization +# these can just be use with pydantic type-hints. + +# Freezes without validating contents. Use FrozenDictType below when the +# declared value types must still be enforced. +frozen_dict_validator = PlainValidator(lambda x: FrozenDict(x)) +frozen_dict_serializer = PlainSerializer(lambda x: dict(x)) + +# A datetime64 +DateTime64 = Annotated[ + np.datetime64, + PlainValidator(to_datetime64), + PlainSerializer(to_str, when_used="json"), # getting undefined name +] + +TimeDelta64 = Annotated[ + np.timedelta64, + PlainValidator(to_timedelta64), + PlainSerializer(to_str, when_used="json"), # getting undefined name +] + +# The validator may preserve non-numpy array-likes (see compat.array), but +# ndarray is deliberately the single static face of array values; a structural +# protocol is not worth the complexity it spreads through every signature. +ArrayLike = Annotated[ + np.ndarray, + PlainValidator(array), +] + +DTypeLike = Annotated[ + str, + PlainValidator(np.dtype), +] + + +def _to_unit_quantity(value): + """Read units, refusing a quantity that carries many magnitudes.""" + out = get_quantity(value) + try: + # Passing a sequence makes pint build an array magnitude, which is + # writable through the frozen model holding it. Asking whether it can + # be hashed is the direct question; no real unit spelling fails it. + hash(out) + except TypeError: + msg = f"Units must name a single unit, got {value!r}." + raise ValueError(msg) from None + return out + + +UnitQuantity = Annotated[ + Quantity | str | None, + PlainValidator(_to_unit_quantity), + PlainSerializer(get_quantity_str), +] + +CommaSeparatedStr = Annotated[ + str, PlainValidator(lambda x: x if isinstance(x, str) else ",".join(x)) +] + +K = TypeVar("K") +V = TypeVar("V") + +# Mapping, not dict: the runtime value is a FrozenDict, so a dict annotation +# would let type checkers pass writes that raise. AfterValidator, not +# PlainValidator: a plain validator would replace the declared value-type check. +FrozenDictType = Annotated[ + Mapping[K, V], + AfterValidator(lambda x: FrozenDict(x)), + PlainSerializer(dict), +] + +UTF8Str = Annotated[str, PlainValidator(unbyte)] + +# A positive (> 0) integer. +PositiveInt = Annotated[int, Field(gt=0)] + +# A float which must be finite; nan/inf silently poison downstream math. +FiniteFloat = Annotated[float, Field(allow_inf_nan=False)] + + +def _none_if_not_finite(value): + """Read a non-finite number as an absent one.""" + if value is None: + return None + try: + return value if np.isfinite(value) else None + except (TypeError, ValueError): + # Not a number at all. Whatever it is, the field's own validation + # is what should describe it. + return value + + +# How to spell a number which may be absent. Absence is `None`, because +# json has no spelling for nan: a nan-defaulted float writes `null` and +# then refuses to read it back, so its class cannot reconstruct from its +# own dump. A file which spells "unknown" as nan is read as absent rather +# than refused, since that is the same statement in another notation. +OptionalFiniteFloat = Annotated[ + FiniteFloat | None, BeforeValidator(_none_if_not_finite) +] + +# A positive (> 0), finite (no nan/inf) float. +PositiveFiniteFloat = Annotated[float, Field(gt=0, allow_inf_nan=False)] diff --git a/dascore/proc/basic.py b/dascore/proc/basic.py index 5254a4996..9aed6236f 100644 --- a/dascore/proc/basic.py +++ b/dascore/proc/basic.py @@ -19,9 +19,9 @@ ) from dascore.core.coords import get_coord from dascore.exceptions import ParameterError +from dascore.models import ArrayLike from dascore.utils.array import _apply_binary_ufunc from dascore.utils.misc import _get_nullish -from dascore.utils.models import ArrayLike from dascore.utils.patch import ( align_patch_coords, get_dim_axis_value, diff --git a/dascore/proc/inventory.py b/dascore/proc/inventory.py index 7c26654b9..dee32c061 100644 --- a/dascore/proc/inventory.py +++ b/dascore/proc/inventory.py @@ -35,12 +35,12 @@ PatchError, UnresolvedPatchError, ) +from dascore.models import values_equal from dascore.proc.coords import update_coords from dascore.units import get_quantity_str from dascore.utils.attrs import validate_conflict from dascore.utils.docs import compose_docstring from dascore.utils.misc import iterate, validate_acquisition_key -from dascore.utils.models import values_equal from dascore.utils.patch import patch_function from dascore.utils.time import to_datetime64 diff --git a/dascore/proc/mute.py b/dascore/proc/mute.py index 04bbbd221..9d9d4d9c8 100644 --- a/dascore/proc/mute.py +++ b/dascore/proc/mute.py @@ -9,16 +9,17 @@ import numpy as np from numpy.linalg import norm from numpy.typing import NDArray +from pydantic import BaseModel, ConfigDict from scipy.ndimage import gaussian_filter import dascore as dc from dascore.constants import PatchType from dascore.exceptions import ParameterError +from dascore.models import sensible_model_equals, sensible_model_hash from dascore.utils.docs import compose_docstring from dascore.utils.misc import ( get_2d_line_intersection, ) -from dascore.utils.models import DascoreBaseModel from dascore.utils.patch import get_dim_axis_value, patch_function _smooth_param = """ @@ -40,11 +41,23 @@ _smooth_type = None | float | int | tuple[float | int, ...] | dict[str, float | int] -class _MuteGeometry(ABC, DascoreBaseModel): +class _MuteGeometry(ABC, BaseModel): """ Parent class for Mute Geometry. + + A plain pydantic model: these carry a mute's geometry from argument + parsing to mask construction and are never serialized. Subclassing + DascoreBaseModel would only claim each a tag in the model registry, + naming them in documents they never appear in. """ + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + # These hold ndarrays, which pydantic's generated __eq__ cannot compare + # (it raises on the truth value of an array), so they keep DASCore's. + __eq__ = sensible_model_equals + __hash__ = sensible_model_hash + dims: tuple[str, ...] axes: tuple[int, ...] relative: bool = True diff --git a/dascore/utils/array.py b/dascore/utils/array.py index 51bd20185..19412c630 100644 --- a/dascore/utils/array.py +++ b/dascore/utils/array.py @@ -15,9 +15,9 @@ from dascore.compat import array, is_array from dascore.constants import DEFAULT_ATTRS_TO_IGNORE, PatchType from dascore.exceptions import ParameterError, PatchBroadcastError, UnitError +from dascore.models import ArrayLike from dascore.units import DimensionalityError, Quantity, Unit, get_quantity from dascore.utils.misc import iterate -from dascore.utils.models import ArrayLike from dascore.utils.patch import ( _merge_aligned_coords, _merge_models, diff --git a/dascore/utils/coordmanager.py b/dascore/utils/coordmanager.py index 504a7e520..e176aebe3 100644 --- a/dascore/utils/coordmanager.py +++ b/dascore/utils/coordmanager.py @@ -10,8 +10,8 @@ import dascore as dc from dascore.exceptions import CoordMergeError +from dascore.models import ArrayLike from dascore.utils.display import get_nice_text -from dascore.utils.models import ArrayLike def merge_coord_managers( diff --git a/dascore/utils/models.py b/dascore/utils/models.py index 64d67c95d..aa4ddfca5 100644 --- a/dascore/utils/models.py +++ b/dascore/utils/models.py @@ -1,305 +1,16 @@ -"""Utilities for models.""" +""" +Deprecated home of DASCore's models; import from [dascore.models](`dascore.models`). -from __future__ import annotations - -from collections.abc import Mapping -from functools import cached_property -from typing import Annotated, TypeVar - -import numpy as np -import pandas as pd -from pydantic import ( - AfterValidator, - BaseModel, - ConfigDict, - Field, - PlainSerializer, - PlainValidator, - model_validator, -) -from typing_extensions import Self - -from dascore.compat import array, is_array_like -from dascore.exceptions import InvalidInventoryError -from dascore.units import Quantity, get_quantity, get_quantity_str -from dascore.utils.mapping import FrozenDict -from dascore.utils.misc import _all_null, all_close, to_str, unbyte -from dascore.utils.time import to_datetime64, to_timedelta64 - -# --- A list of custom types with appropriate serialization/deserialization -# these can just be use with pydantic type-hints. - -# Freezes without validating contents. Use FrozenDictType below when the -# declared value types must still be enforced. -frozen_dict_validator = PlainValidator(lambda x: FrozenDict(x)) -frozen_dict_serializer = PlainSerializer(lambda x: dict(x)) - -# A datetime64 -DateTime64 = Annotated[ - np.datetime64, - PlainValidator(to_datetime64), - PlainSerializer(to_str, when_used="json"), # getting undefined name -] - -TimeDelta64 = Annotated[ - np.timedelta64, - PlainValidator(to_timedelta64), - PlainSerializer(to_str, when_used="json"), # getting undefined name -] - -# The validator may preserve non-numpy array-likes (see compat.array), but -# ndarray is deliberately the single static face of array values; a structural -# protocol is not worth the complexity it spreads through every signature. -ArrayLike = Annotated[ - np.ndarray, - PlainValidator(array), -] - -DTypeLike = Annotated[ - str, - PlainValidator(np.dtype), -] - - -def _to_unit_quantity(value): - """Read units, refusing a quantity that carries many magnitudes.""" - out = get_quantity(value) - try: - # Passing a sequence makes pint build an array magnitude, which is - # writable through the frozen model holding it. Asking whether it can - # be hashed is the direct question; no real unit spelling fails it. - hash(out) - except TypeError: - msg = f"Units must name a single unit, got {value!r}." - raise ValueError(msg) from None - return out - - -UnitQuantity = Annotated[ - Quantity | str | None, - PlainValidator(_to_unit_quantity), - PlainSerializer(get_quantity_str), -] - -CommaSeparatedStr = Annotated[ - str, PlainValidator(lambda x: x if isinstance(x, str) else ",".join(x)) -] - -K = TypeVar("K") -V = TypeVar("V") - -# Mapping, not dict: the runtime value is a FrozenDict, so a dict annotation -# would let type checkers pass writes that raise. AfterValidator, not -# PlainValidator: a plain validator would replace the declared value-type check. -FrozenDictType = Annotated[ - Mapping[K, V], - AfterValidator(lambda x: FrozenDict(x)), - PlainSerializer(dict), -] - -UTF8Str = Annotated[str, PlainValidator(unbyte)] - -# A positive (> 0) integer. -PositiveInt = Annotated[int, Field(gt=0)] - -# A positive (> 0), finite (no nan/inf) float. -PositiveFiniteFloat = Annotated[float, Field(gt=0, allow_inf_nan=False)] - - -def sensible_model_equals(self: BaseModel | Mapping, other: object) -> bool: - """Custom equality to not compare private attrs and handle numpy arrays.""" - d1 = self.model_dump() if isinstance(self, BaseModel) else self - if isinstance(other, BaseModel): - d2 = other.model_dump() - elif isinstance(other, Mapping): - d2 = other - else: # nothing else can carry the same fields - return NotImplemented - if not set(d1) == set(d2): # different keys, not equal - return False - for name in set(x for x in d1 if not x.startswith("_")): - # skip any private attributes. - if not values_equal(d1[name], d2[name]): - return False - return True - - -def values_equal(val1, val2) -> bool: - """Recursively compare dumped values; nulls are equal only to nulls.""" - if is_array_like(val1) or is_array_like(val2): - arr1, arr2 = np.asarray(val1), np.asarray(val2) - if arr1.shape != arr2.shape: - return False - if not np.array_equal(pd.isnull(arr1), pd.isnull(arr2)): - return False - return bool(all_close(arr1, arr2)) - if isinstance(val1, Mapping) and isinstance(val2, Mapping): - if set(val1) != set(val2): - return False - return all(values_equal(val1[key], val2[key]) for key in val1) - if isinstance(val1, list | tuple) and isinstance(val2, list | tuple): - if len(val1) != len(val2): - return False - return all(values_equal(v1, v2) for v1, v2 in zip(val1, val2)) - return bool(val1 == val2 or (_all_null(val1) and _all_null(val2))) +Everything here is re-exported from its new home so out-of-tree readers which +import from this path keep working. Re-exported wholesale rather than listed, +since a second hand-maintained list is a second thing to forget. +""" +from __future__ import annotations -def _hash_key(value): - """Map a value onto one that hashes the way values_equal compares.""" - if value is None or isinstance(value, str | int): - return value - # Nulls count as equal to one another above, but every nan and NaT is a - # fresh object and both hash by identity, so they collapse to one key. - if isinstance(value, float): - return None if value != value else value - if isinstance(value, np.datetime64 | np.timedelta64): - return None if np.isnat(value) else value - # Mappings are compared without regard to order. - if isinstance(value, Mapping): - return frozenset((k, _hash_key(v)) for k, v in value.items()) - if isinstance(value, tuple): - return tuple(_hash_key(v) for v in value) - return value - - -def sensible_model_hash(self: BaseModel) -> int: - """Hash a model on its fields, agreeing with sensible_model_equals.""" - # Keyed by name and unordered, because equality compares the field names - # it finds rather than the order they were declared in. - fields = type(self).model_fields - return hash(frozenset((x, _hash_key(getattr(self, x))) for x in fields)) - - -class DascoreBaseModel(BaseModel): - """A base model with sensible configurations.""" - - _cache = {} - - model_config = ConfigDict( - extra="ignore", # TODO: change to raise, then let subclass overwrite - validate_assignment=True, - ignored_types=(cached_property,), - frozen=True, - validate_default=True, - arbitrary_types_allowed=True, - ) - - def new(self, **kwargs) -> Self: - """Create new instance with some attributed updated.""" - out = self.model_dump(exclude_unset=True) - out.update(kwargs) - return self.__class__(**out) - - @classmethod - def get_summary_df(cls): - """Get dataframe of attributes and descriptions for display.""" - fields = cls.model_fields - names_desc = { - i: v.description - for i, v in fields.items() - if getattr(v, "description", False) - } - out = pd.Series(names_desc).to_frame(name="description") - out.index.name = "attribute" - return out - - __eq__ = sensible_model_equals - # Defined together: pydantic would otherwise derive a hash straight from - # the field values, which disagrees with how __eq__ treats nulls. - __hash__ = sensible_model_hash - - -class InventoryModel(DascoreBaseModel): - """ - Base class for immutable DASDAE inventory objects. - - Every inventory object carries two uniform attachment points for - information the model does not otherwise represent: ``description`` - (free prose for humans, matching StationXML's Description element) - and ``extra_fields`` (typed key-values, e.g. for round-tripping - unmodeled metadata from external formats). - - Every field is immutable: collections are tuples and mappings are - frozen, so instances are safe to hold by reference. They hash on their - field values whenever those values are themselves hashable. - """ - - description: str = Field(default="", description="Free-text description.") - extra_fields: FrozenDictType[str, str | int | float | bool] = Field( - default_factory=dict, - description="Extra metadata not represented by standardized fields.", - ) - - model_config = ConfigDict( - extra="forbid", - frozen=True, - validate_assignment=True, - validate_default=True, - arbitrary_types_allowed=True, - ) - - def new(self, **kwargs) -> Self: - """ - Create a new instance with some attributes updated. - - Dumps all fields (not just the set ones) so union discriminators and - validator-normalized fields survive reconstruction. - """ - out = self.model_dump() - out.update(kwargs) - return self.__class__(**out) - - -class TimeRangedModel(InventoryModel): - """Base class for inventory objects with time-validity epochs. - - Validity intervals are half-open, ``[start_time, end_time)``; an unset - (NaT) end time means the epoch is ongoing. All times are UTC. - """ - - start_time: DateTime64 = Field( - default=np.datetime64("NaT", "ns"), - description="Start time for which this metadata item is valid (UTC).", - ) - end_time: DateTime64 = Field( - default=np.datetime64("NaT", "ns"), - description=( - "End time for which this metadata item is valid (UTC); NaT while ongoing." - ), - ) - - @model_validator(mode="after") - def _check_time_order(self): - """A set end time must follow the start time.""" - start, end = self.start_time, self.end_time - if not pd.isnull(start) and not pd.isnull(end) and end <= start: - msg = f"end_time {end} must be after start_time {start}." - raise InvalidInventoryError(msg) - return self - - def is_effective_at(self, time) -> bool: - """Return True if this epoch is valid at the supplied time (half-open).""" - time = to_datetime64(time) - if pd.isnull(time): - return True - start = self.start_time - end = self.end_time - after_start = pd.isnull(start) or start <= time - before_end = pd.isnull(end) or time < end - return bool(after_start and before_end) +from pydantic import BaseModel - def overlaps(self, other: TimeRangedModel) -> bool: - """ - Return True if two half-open validity intervals overlap. +from dascore.models import * # noqa: F403 +from dascore.models import __all__ as _models_all - Unset (NaT) starts are unbounded past; unset ends are ongoing. - """ - s1, e1, s2, e2 = ( - self.start_time, - self.end_time, - other.start_time, - other.end_time, - ) - first_starts_before = pd.isnull(e2) or pd.isnull(s1) or s1 < e2 - second_starts_before = pd.isnull(e1) or pd.isnull(s2) or s2 < e1 - return bool(first_starts_before and second_starts_before) +__all__ = [*_models_all, "BaseModel"] diff --git a/dascore/utils/plugins.py b/dascore/utils/plugins.py index 11cf1e007..6a9437232 100644 --- a/dascore/utils/plugins.py +++ b/dascore/utils/plugins.py @@ -7,6 +7,9 @@ from importlib.metadata import entry_points from typing import Any +# The entry-point group every FiberIO plugin registers under. +FIBER_IO_GROUP = "dascore.fiber_io" + @functools.cache def get_entry_point_loaders(entry_point_group: str) -> dict[str, Any]: diff --git a/docs/contributing/new_format.qmd b/docs/contributing/new_format.qmd index c27c28f4f..c95a7a490 100644 --- a/docs/contributing/new_format.qmd +++ b/docs/contributing/new_format.qmd @@ -192,7 +192,10 @@ name and who may change it later. whatever name the format uses. These are welcome, but each new one must be listed in the `VENDOR_ATTRS` set in `tests/test_io/test_common_io.py`, which is where a reviewer checks that the value is not one of the facts - above wearing a vendor's name. + above wearing a vendor's name. Declaring them on a `PatchAttrs` subclass + gets them validated and coerced; see + [format-specific subclasses](`docs/notes/patch_attrs.qmd`) for how one is + spelled, and why an optional number is never defaulted to nan or inf. Stay as close to the file's own spelling as the rules above allow. A value that is one of the facts in (2) has to take the canonical name, because two diff --git a/docs/notes/patch_attrs.qmd b/docs/notes/patch_attrs.qmd index 3977949d9..483de5b9b 100644 --- a/docs/notes/patch_attrs.qmd +++ b/docs/notes/patch_attrs.qmd @@ -39,3 +39,47 @@ DASCore-assigned `data_type` values should be snake_case and listed in `VALID_DA | `data_type="some_value"` | Set the returned patch's `data_type` to that value. | Functions may still require a specific input label with `required_attrs`, for example `required_attrs={"data_type": "velocity"}`. This should only be used when the function's assumptions truly depend on that label and are documented. + +## Format-specific subclasses + +A format whose files carry fields the base class does not model declares a +`PatchAttrs` subclass for them; there are more than a dozen under +`dascore/io/`. Since the fields are declared, they are validated and coerced +like any other, rather than surviving as untyped extras. + +A serialized patch names the class it holds, so the subclass comes back: + +```python +import dascore as dc +from dascore.io.odh4.core import ODH4PatchAttrs + +patch = dc.get_example_patch() +patch = patch.update(attrs=ODH4PatchAttrs(**dict(patch.attrs), gauge_length=10.0)) +patch.io.write("example.h5", "dasdae") + +type(dc.read("example.h5")[0].attrs) # ODH4PatchAttrs +``` + +The name is a registered one rather than an import path, so moving a class +between modules does not break files already written, and reading a file +never imports a path it names. A file naming a class which is not installed +— written, say, by a plugin this environment lacks — reads as the base +`PatchAttrs` with a warning rather than failing. + +### Spelling an optional number + +An absent number is spelled `OptionalFiniteFloat`, defaulting to `None`: + +```python +from dascore.models import OptionalFiniteFloat + +class JingleV1PatchAttrs(dc.PatchAttrs): + """Attrs for the jingle format.""" + + gauge_length: OptionalFiniteFloat = None +``` + +Not `float = np.nan`, and not `float = np.inf`. JSON spells neither, so such +a field writes `null` and then refuses to read it back, which means the class +cannot reconstruct from its own `model_dump_json()`. A non-finite value read +from a file is taken as an absent one rather than refused. diff --git a/docs/tutorial/patch.qmd b/docs/tutorial/patch.qmd index 1415a9a03..9e7b9b33c 100644 --- a/docs/tutorial/patch.qmd +++ b/docs/tutorial/patch.qmd @@ -218,7 +218,7 @@ you intend to select literally. ## Attrs -The metadata stored in `Patch.attrs` is a [pydantic model](https://docs.pydantic.dev/usage/models/) which enforces a schema and provides validation. [`PatchAttrs.get_summary_df`](`dascore.utils.models.DascoreBaseModel.get_summary_df`) generates a table of the attribute descriptions: +The metadata stored in `Patch.attrs` is a [pydantic model](https://docs.pydantic.dev/usage/models/) which enforces a schema and provides validation. [`PatchAttrs.get_summary_df`](`dascore.models.base.DascoreBaseModel.get_summary_df`) generates a table of the attribute descriptions: ```{python} diff --git a/tests/test_core/test_inventory.py b/tests/test_core/test_inventory.py index 77caba3d3..f4df337b2 100644 --- a/tests/test_core/test_inventory.py +++ b/tests/test_core/test_inventory.py @@ -14,8 +14,8 @@ from dascore.core import Inventory from dascore.core import inventory as inv from dascore.exceptions import InvalidInventoryError +from dascore.models import values_equal from dascore.utils.mapping import FrozenDict -from dascore.utils.models import values_equal def build_inventory() -> inv.Inventory: @@ -874,7 +874,7 @@ def test_keyless_dict_resource_adopts_key(self): """A dict resource without resource_id adopts its pool key.""" pytest.importorskip("yaml") inventory = inv.Inventory( - resources={"cab-1": {"type": "Cable", "name": "mycable"}} + resources={"cab-1": {"object_type": "Cable", "name": "mycable"}} ) assert inventory.get_resource("cab-1").resource_id == "cab-1" loaded = inv.Inventory.from_yaml(inventory.to_yaml()) @@ -1130,22 +1130,22 @@ def test_values_equal_branches(self): assert values_equal((1.0, np.nan), (1.0, np.nan)) -class TestTypeTag: - """The serialization type tag is invisible to users.""" +class TestObjectTypeTag: + """The union members' own tag field is invisible to users.""" def test_hidden_from_repr(self): """Hidden from repr.""" cable = inv.Cable(resource_id="c1", name="c") - assert "type" not in repr(cable) + assert "object_type" not in repr(cable) def test_present_in_dump(self): - """Present in dump.""" - assert inv.Cable(resource_id="c1").model_dump()["type"] == "Cable" + """Present in dump, unlike the tag every other model is given.""" + assert inv.Cable(resource_id="c1").model_dump()["object_type"] == "Cable" def test_wrong_tag_rejected(self): """Wrong tag rejected.""" with pytest.raises(ValidationError): - inv.Cable(resource_id="c1", type="Enclosure") + inv.Cable(resource_id="c1", object_type="Enclosure") class TestUniformAttachments: @@ -1241,7 +1241,7 @@ def test_frozen_record_disagreeing_with_its_key_is_refused(self): """A record is read as a record whatever kind of mapping it is.""" # Read as an object instead, its resource_id goes unseen and the # mismatch only surfaces on the next load. - record = FrozenDict({"type": "Cable", "resource_id": "elsewhere"}) + record = FrozenDict({"object_type": "Cable", "resource_id": "elsewhere"}) with pytest.raises(ValidationError, match="disagrees with resource_id"): inv.Inventory(resources={"cable-01": record}) diff --git a/tests/test_core/test_inventory_loader.py b/tests/test_core/test_inventory_loader.py index 8bf316f1e..3f0feb56f 100644 --- a/tests/test_core/test_inventory_loader.py +++ b/tests/test_core/test_inventory_loader.py @@ -8,6 +8,7 @@ import numpy as np import pytest +from pydantic import ValidationError import dascore as dc from dascore.core import inventory as inv @@ -16,14 +17,15 @@ InvalidInventoryError, MissingOptionalDependencyError, ) -from dascore.utils.models import InventoryModel, TimeRangedModel +from dascore.models import InventoryModel, TimeRangedModel +from dascore.models.registry import TAG_FIELD pytest.importorskip("yaml") # A minimal directory which loads: one acquisition names everything above it. MINIMAL = { - "acquisitions/DAS.L001..RAW.yaml": "type: Acquisition\ndata_category: DAS\n", + "acquisitions/DAS.L001..RAW.yaml": "object_type: Acquisition\ndata_category: DAS\n", } @@ -116,7 +118,7 @@ def test_resolves_like_any_inventory(self, make_inventory): out = make_inventory( { "acquisitions/DAS.L001..RAW.yaml": ( - "type: Acquisition\ndata_category: DAS\ngauge_length: 10.0\n" + "object_type: Acquisition\ndata_category: DAS\ngauge_length: 10.0\n" ), } ) @@ -126,15 +128,16 @@ def test_full_directory(self, make_inventory): """Every container contributes to one inventory.""" out = make_inventory( { - "inventory.yaml": "type: Inventory\nschema_version: 1\n", + "inventory.yaml": "object_type: Inventory\nschema_version: 1\n", "resources/int_01.yaml": ( - "type: Interrogator\nmanufacturer: Fake\nmodel: FI-1\n" + "object_type: Interrogator\nmanufacturer: Fake\nmodel: FI-1\n" ), - "networks/DAS.yaml": "type: Network\nname: test network\n", - "fiber_arrays/DAS.L001.yaml": "type: FiberArray\nname: first\n", - "stations/DAS.STA1.yaml": "type: Station\nname: a station\n", + "networks/DAS.yaml": "object_type: Network\nname: test network\n", + "fiber_arrays/DAS.L001.yaml": "object_type: FiberArray\nname: first\n", + "stations/DAS.STA1.yaml": "object_type: Station\nname: a station\n", "acquisitions/DAS.L001..RAW.yaml": ( - "type: Acquisition\ndata_category: DAS\ninterrogator: int_01\n" + "object_type: Acquisition\ndata_category: DAS\n" + "interrogator: int_01\n" ), } ) @@ -152,7 +155,8 @@ def test_json_and_yaml_are_interchangeable(self, tmp_path): tmp_path / "yaml_form", { "acquisitions/DAS.L001..RAW.yaml": ( - "type: Acquisition\ndata_category: DAS\ngauge_length: 4.0\n" + "object_type: Acquisition\ndata_category: DAS\n" + "gauge_length: 4.0\n" ) }, ) @@ -162,7 +166,7 @@ def test_json_and_yaml_are_interchangeable(self, tmp_path): tmp_path / "json_form", { "acquisitions/DAS.L001..RAW.json": ( - '{"type": "Acquisition", "data_category": "DAS", ' + '{"object_type": "Acquisition", "data_category": "DAS", ' '"gauge_length": 4.0}' ) }, @@ -172,7 +176,9 @@ def test_json_and_yaml_are_interchangeable(self, tmp_path): def test_yml_suffix(self, make_inventory): """The short YAML suffix is the same spelling.""" - out = make_inventory({"acquisitions/DAS.L001..RAW.yml": "type: Acquisition\n"}) + out = make_inventory( + {"acquisitions/DAS.L001..RAW.yml": "object_type: Acquisition\n"} + ) assert out.networks[0].fiber_arrays[0].acquisitions[0].code == "RAW" def test_envelope_is_optional(self, make_inventory): @@ -186,7 +192,7 @@ def test_envelope_states_the_singletons(self, make_inventory): { **MINIMAL, "inventory.yaml": ( - "type: Inventory\n" + "object_type: Inventory\n" "resource_id: my-inventory\n" "coordinate_reference_system:\n" " authority: EPSG\n" @@ -204,7 +210,7 @@ def test_entity_directory_form(self, make_inventory): { **MINIMAL, "fiber_arrays/DAS.L001/attrs.yaml": ( - "type: FiberArray\nname: from a directory\n" + "object_type: FiberArray\nname: from a directory\n" ), } ) @@ -221,7 +227,7 @@ def test_non_participating_files_are_ignored(self, make_inventory): "notes/scratch.yaml": "just: a note\n", # Nor is one which does not parse at all. "notes/broken.yaml": "{[unclosed\n", - ".hidden/DAS.yaml": "type: Network\n", + ".hidden/DAS.yaml": "object_type: Network\n", "acquisitions/README.txt": "a note beside the objects\n", } ) @@ -232,7 +238,7 @@ def test_resource_id_may_hold_dots(self, make_inventory): out = make_inventory( { **MINIMAL, - "resources/cable.01.yaml": "type: Cable\nname: a cable\n", + "resources/cable.01.yaml": "object_type: Cable\nname: a cable\n", } ) assert out.get_resource("cable.01").name == "a cable" @@ -242,7 +248,7 @@ def test_restated_address_may_agree(self, make_inventory): out = make_inventory( { "acquisitions/DAS.L001.01.RAW.yaml": ( - "type: Acquisition\ncode: RAW\nlocation_code: '01'\n" + "object_type: Acquisition\ncode: RAW\nlocation_code: '01'\n" ) } ) @@ -256,7 +262,7 @@ class TestEpochNames: def test_date_only_means_midnight_utc(self, make_inventory): """A date-only suffix is valid when that precision suffices.""" out = make_inventory( - {"acquisitions/DAS.L001..RAW@2024-06-01.yaml": "type: Acquisition\n"} + {"acquisitions/DAS.L001..RAW@2024-06-01.yaml": "object_type: Acquisition\n"} ) acquisition = out.networks[0].fiber_arrays[0].acquisitions[0] assert acquisition.start_time == np.datetime64("2024-06-01T00:00:00", "ns") @@ -266,7 +272,7 @@ def test_basic_time_and_fractional_seconds(self, make_inventory): out = make_inventory( { "acquisitions/DAS.L001..RAW@2024-05-12T103000.12.yaml": ( - "type: Acquisition\n" + "object_type: Acquisition\n" ) } ) @@ -279,7 +285,7 @@ def test_suffix_agrees_with_stated_start(self, make_inventory): out = make_inventory( { "acquisitions/DAS.L001..RAW@2024-06-01.yaml": ( - "type: Acquisition\nstart_time: '2024-06-01'\n" + "object_type: Acquisition\nstart_time: '2024-06-01'\n" ) } ) @@ -291,10 +297,11 @@ def test_epochs_of_one_acquisition(self, make_inventory): out = make_inventory( { "acquisitions/DAS.L001..RAW.yaml": ( - "type: Acquisition\nend_time: '2024-06-01'\ngauge_length: 10.0\n" + "object_type: Acquisition\nend_time: '2024-06-01'\n" + "gauge_length: 10.0\n" ), "acquisitions/DAS.L001..RAW@2024-06-01.yaml": ( - "type: Acquisition\ngauge_length: 5.0\n" + "object_type: Acquisition\ngauge_length: 5.0\n" ), } ) @@ -308,12 +315,14 @@ def test_acquisition_lands_in_its_array_epoch(self, make_inventory): out = make_inventory( { "fiber_arrays/DAS.L001.yaml": ( - "type: FiberArray\nname: first\nend_time: '2024-06-01'\n" + "object_type: FiberArray\nname: first\nend_time: '2024-06-01'\n" ), "fiber_arrays/DAS.L001@2024-06-01.yaml": ( - "type: FiberArray\nname: second\n" + "object_type: FiberArray\nname: second\n" + ), + "acquisitions/DAS.L001..RAW@2024-07-01.yaml": ( + "object_type: Acquisition\n" ), - "acquisitions/DAS.L001..RAW@2024-07-01.yaml": "type: Acquisition\n", } ) arrays = {x.name: x for x in out.networks[0].fiber_arrays} @@ -326,37 +335,39 @@ class TestNearMisses: def test_typo_in_container_name(self, make_inventory): """A typo must not quietly load an inventory with no acquisitions.""" - files = {"aquisitions/DAS.L001..RAW.yaml": "type: Acquisition\n"} + files = {"aquisitions/DAS.L001..RAW.yaml": "object_type: Acquisition\n"} with pytest.raises(InvalidInventoryError, match="nothing contains it"): make_inventory(files) def test_model_file_at_the_root(self, make_inventory): """An object at the root is outside every container.""" - files = {**MINIMAL, "DAS.L001.yaml": "type: FiberArray\n"} + files = {**MINIMAL, "DAS.L001.yaml": "object_type: FiberArray\n"} with pytest.raises(InvalidInventoryError, match="nothing contains it"): make_inventory(files) def test_missing_type(self, make_inventory): """A file which does not say what it is has not participated.""" files = {"acquisitions/DAS.L001..RAW.yaml": "data_category: DAS\n"} - with pytest.raises(InvalidInventoryError, match="declares no type"): + with pytest.raises(InvalidInventoryError, match="declares no object_type"): make_inventory(files) def test_wrong_container(self, make_inventory): """The container checks the declared type rather than supplying it.""" - files = {"fiber_arrays/DAS.L001.yaml": "type: Acquisition\n"} + files = {"fiber_arrays/DAS.L001.yaml": "object_type: Acquisition\n"} with pytest.raises(InvalidInventoryError, match="cannot hold"): make_inventory(files) def test_unknown_type(self, make_inventory): """A type which names no model is unknown rather than misfiled.""" - files = {"fiber_arrays/DAS.L001.yaml": "type: Telescope\n"} + files = {"fiber_arrays/DAS.L001.yaml": "object_type: Telescope\n"} with pytest.raises(InvalidInventoryError, match="unknown"): make_inventory(files) def test_restated_address_disagrees(self, make_inventory): """There is never a precedence rule between two spellings.""" - files = {"acquisitions/DAS.L001..RAW.yaml": "type: Acquisition\ncode: DEC\n"} + files = { + "acquisitions/DAS.L001..RAW.yaml": "object_type: Acquisition\ncode: DEC\n" + } with pytest.raises(InvalidInventoryError, match="must agree with the name"): make_inventory(files) @@ -364,7 +375,7 @@ def test_restated_start_time_disagrees(self, make_inventory): """The epoch suffix is a restated address, so it must agree.""" files = { "acquisitions/DAS.L001..RAW@2024-06-01.yaml": ( - "type: Acquisition\nstart_time: '2024-06-02'\n" + "object_type: Acquisition\nstart_time: '2024-06-02'\n" ) } with pytest.raises(InvalidInventoryError, match="must agree with the name"): @@ -378,7 +389,7 @@ def test_restated_start_time_disagrees(self, make_inventory): ) def test_illegal_address_token_names_the_file(self, make_inventory, name, level): """The entity a token names is built from every address, not one file.""" - files = {f"acquisitions/{name}": "type: Acquisition\n"} + files = {f"acquisitions/{name}": "object_type: Acquisition\n"} with pytest.raises(InvalidInventoryError, match=f"names {level}") as info: make_inventory(files) # The point of the check is which file to open, so assert the name. @@ -386,7 +397,7 @@ def test_illegal_address_token_names_the_file(self, make_inventory, name, level) def test_wrong_token_count(self, make_inventory): """An acquisition name is an address of four tokens.""" - files = {"acquisitions/DAS.L001.RAW.yaml": "type: Acquisition\n"} + files = {"acquisitions/DAS.L001.RAW.yaml": "object_type: Acquisition\n"} with pytest.raises(InvalidInventoryError, match="address of 4"): make_inventory(files) @@ -394,7 +405,7 @@ def test_schema_version_outside_the_envelope(self, make_inventory): """The envelope versions the document exactly once.""" files = { "acquisitions/DAS.L001..RAW.yaml": ( - "type: Acquisition\nschema_version: 1\n" + "object_type: Acquisition\nschema_version: 1\n" ) } with pytest.raises(InvalidInventoryError, match="envelope versions"): @@ -404,7 +415,7 @@ def test_invalid_field_names_the_file(self, make_inventory): """A model error says which file could not be read.""" files = { "acquisitions/DAS.L001..RAW.yaml": ( - "type: Acquisition\ngauge_length: not a number\n" + "object_type: Acquisition\ngauge_length: not a number\n" ) } with pytest.raises(InvalidInventoryError, match=r"DAS\.L001\.\.RAW\.yaml"): @@ -435,9 +446,9 @@ class TestOverlookedInput: def test_object_filed_inside_an_entity_directory(self, make_inventory): """An object one level too deep must not be silently dropped.""" files = { - "fiber_arrays/DAS.L001/attrs.yaml": "type: FiberArray\n", + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", "fiber_arrays/DAS.L001/misplaced/DAS.L001..RAW.yaml": ( - "type: Acquisition\n" + "object_type: Acquisition\n" ), } with pytest.raises(InvalidInventoryError, match="nothing contains it"): @@ -447,9 +458,9 @@ def test_object_filed_inside_an_entity_directory(self, make_inventory): def test_upper_case_suffixes(self, make_inventory, suffix): """A case-insensitive filesystem holds one file, not two spellings.""" text = ( - '{"type": "Acquisition"}' + '{"object_type": "Acquisition"}' if suffix == "JSON" - else "type: Acquisition\ndata_category: DAS\n" + else "object_type: Acquisition\ndata_category: DAS\n" ) out = make_inventory({f"acquisitions/DAS.L001..RAW.{suffix}": text}) assert out.networks[0].fiber_arrays[0].acquisitions[0].code == "RAW" @@ -457,7 +468,10 @@ def test_upper_case_suffixes(self, make_inventory, suffix): def test_upper_case_envelope(self, make_inventory): """The envelope is found however its suffix is spelled.""" out = make_inventory( - {**MINIMAL, "inventory.YAML": "type: Inventory\nresource_id: shouted\n"} + { + **MINIMAL, + "inventory.YAML": "object_type: Inventory\nresource_id: shouted\n", + } ) assert out.resource_id == "shouted" @@ -465,12 +479,12 @@ def test_child_outliving_its_parent_epoch(self, make_inventory): """Starting inside an epoch is not enough to belong to it.""" files = { "fiber_arrays/DAS.L001.yaml": ( - "type: FiberArray\nend_time: '2024-06-01'\n" + "object_type: FiberArray\nend_time: '2024-06-01'\n" ), - "fiber_arrays/DAS.L001@2024-06-01.yaml": "type: FiberArray\n", + "fiber_arrays/DAS.L001@2024-06-01.yaml": "object_type: FiberArray\n", # Starts inside the first epoch and never ends, so resolution # after June would find the second array, which does not hold it. - "acquisitions/DAS.L001..RAW@2024-05-01.yaml": "type: Acquisition\n", + "acquisitions/DAS.L001..RAW@2024-05-01.yaml": "object_type: Acquisition\n", } with pytest.raises(InvalidInventoryError, match="runs past"): make_inventory(files) @@ -480,11 +494,11 @@ def test_child_ending_exactly_at_the_boundary_fits(self, make_inventory): out = make_inventory( { "fiber_arrays/DAS.L001.yaml": ( - "type: FiberArray\nend_time: '2024-06-01'\n" + "object_type: FiberArray\nend_time: '2024-06-01'\n" ), - "fiber_arrays/DAS.L001@2024-06-01.yaml": "type: FiberArray\n", + "fiber_arrays/DAS.L001@2024-06-01.yaml": "object_type: FiberArray\n", "acquisitions/DAS.L001..RAW@2024-05-01.yaml": ( - "type: Acquisition\nend_time: '2024-06-01'\n" + "object_type: Acquisition\nend_time: '2024-06-01'\n" ), } ) @@ -493,7 +507,10 @@ def test_child_ending_exactly_at_the_boundary_fits(self, make_inventory): def test_unreadable_envelope_value_names_the_file(self, make_inventory): """An envelope error reads like every other error this format raises.""" - files = {**MINIMAL, "inventory.yaml": "type: Inventory\nschema_version: nope\n"} + files = { + **MINIMAL, + "inventory.yaml": "object_type: Inventory\nschema_version: nope\n", + } with pytest.raises(InvalidInventoryError, match="Could not read the envelope"): make_inventory(files) @@ -508,17 +525,17 @@ def test_unreadable_envelope_value_names_the_file(self, make_inventory): def test_nested_collections_are_refused(self, make_inventory, where, field, member): """A file may not state what the directory supplies and would replace.""" declared = "Network" if where.startswith("networks") else "FiberArray" - text = f"type: {declared}\n{field}:\n - {member}\n" + text = f"object_type: {declared}\n{field}:\n - {member}\n" with pytest.raises(InvalidInventoryError, match=field): make_inventory({**MINIMAL, where: text}) def test_child_predating_its_parent_epoch(self, make_inventory): """Containment is checked at the near end as well as the far one.""" files = { - "networks/DAS@2024-01-01.yaml": "type: Network\n", + "networks/DAS@2024-01-01.yaml": "object_type: Network\n", # No start of its own, so it claims the unbounded past, which is # outside a network beginning in 2024. - "stations/DAS.STA1.yaml": "type: Station\nend_time: '2020-01-01'\n", + "stations/DAS.STA1.yaml": "object_type: Station\nend_time: '2020-01-01'\n", } with pytest.raises(InvalidInventoryError, match="starts before"): make_inventory(files) @@ -527,11 +544,11 @@ def test_child_ending_after_its_parent_epoch(self, make_inventory): """The far end refuses a real end time, not only an unset one.""" files = { "fiber_arrays/DAS.L001.yaml": ( - "type: FiberArray\nend_time: '2024-06-01'\n" + "object_type: FiberArray\nend_time: '2024-06-01'\n" ), - "fiber_arrays/DAS.L001@2024-06-01.yaml": "type: FiberArray\n", + "fiber_arrays/DAS.L001@2024-06-01.yaml": "object_type: FiberArray\n", "acquisitions/DAS.L001..RAW@2024-05-01.yaml": ( - "type: Acquisition\nend_time: '2024-07-01'\n" + "object_type: Acquisition\nend_time: '2024-07-01'\n" ), } with pytest.raises(InvalidInventoryError, match="runs past"): @@ -549,7 +566,7 @@ def test_symlink_pointing_at_the_inventory(self, tmp_path): def test_envelope_stating_only_its_type(self, tmp_path): """An envelope says the directory is an inventory, whatever it holds.""" root = write_inventory( - tmp_path / "bare", {"inventory.yaml": "type: Inventory\n"} + tmp_path / "bare", {"inventory.yaml": "object_type: Inventory\n"} ) assert not dc.inventory(root).networks @@ -568,7 +585,7 @@ def no_yaml(name, **kwargs): root = write_inventory( tmp_path / "json_only", { - "acquisitions/DAS.L001..RAW.json": '{"type": "Acquisition"}', + "acquisitions/DAS.L001..RAW.json": '{"object_type": "Acquisition"}', # Field material the loader must step over without reading. "notes/log.yaml": "note: a deployment log\n", }, @@ -578,7 +595,7 @@ def no_yaml(name, **kwargs): def test_hidden_object_file_in_a_container(self, make_inventory): """A resource id may hold a dot, but a hidden file names no entity.""" - files = {**MINIMAL, "resources/.cable.yaml": "type: Cable\n"} + files = {**MINIMAL, "resources/.cable.yaml": "object_type: Cable\n"} with pytest.raises(InvalidInventoryError, match="hidden"): make_inventory(files) @@ -586,7 +603,7 @@ def test_epoch_finer_than_a_nanosecond(self, make_inventory): """A name stating more precision than is kept would load as another instant.""" files = { "acquisitions/DAS.L001..RAW@2024-05-12T103000.1234567899.yaml": ( - "type: Acquisition\n" + "object_type: Acquisition\n" ) } with pytest.raises(InvalidInventoryError, match="finer than the nanosecond"): @@ -597,10 +614,11 @@ def test_unquoted_dates(self, make_inventory): out = make_inventory( { "fiber_arrays/DAS.L001.yaml": ( - "type: FiberArray\nstart_time: 2024-01-01\nend_time: 2024-07-01\n" + "object_type: FiberArray\nstart_time: 2024-01-01\n" + "end_time: 2024-07-01\n" ), "acquisitions/DAS.L001..RAW@2024-02-01.yaml": ( - "type: Acquisition\nend_time: 2024-03-01\n" + "object_type: Acquisition\nend_time: 2024-03-01\n" ), } ) @@ -610,8 +628,8 @@ def test_unquoted_dates(self, make_inventory): def test_type_which_is_not_a_name(self, make_inventory): """A type which is not a name at all names no model either.""" - files = {"acquisitions/DAS.L001..RAW.yaml": "type: [Acquisition]\n"} - with pytest.raises(InvalidInventoryError, match="declares no type"): + files = {"acquisitions/DAS.L001..RAW.yaml": "object_type: [Acquisition]\n"} + with pytest.raises(InvalidInventoryError, match="declares no object_type"): make_inventory(files) @@ -621,8 +639,8 @@ class TestOneIdentityOneSpelling: def test_two_extensions(self, make_inventory): """The same name with two extensions is one identity spelled twice.""" files = { - "resources/cable_01.yaml": "type: Cable\n", - "resources/cable_01.json": '{"type": "Cable"}', + "resources/cable_01.yaml": "object_type: Cable\n", + "resources/cable_01.json": '{"object_type": "Cable"}', } with pytest.raises(InvalidInventoryError, match="two extensions"): make_inventory(files) @@ -636,8 +654,8 @@ def test_two_extensions(self, make_inventory): def test_case_only_difference(self, make_inventory): """A case-insensitive filesystem could not hold both.""" files = { - "fiber_arrays/DAS.L001.yaml": "type: FiberArray\n", - "fiber_arrays/das.l001.yaml": "type: FiberArray\n", + "fiber_arrays/DAS.L001.yaml": "object_type: FiberArray\n", + "fiber_arrays/das.l001.yaml": "object_type: FiberArray\n", } with pytest.raises(InvalidInventoryError, match="differ only by case"): make_inventory(files) @@ -655,8 +673,8 @@ def test_case_only_difference_is_named_as_such(self, tmp_path): def test_file_and_directory(self, make_inventory): """Both spellings of one identity at once raise.""" files = { - "fiber_arrays/DAS.L001.yaml": "type: FiberArray\n", - "fiber_arrays/DAS.L001/attrs.yaml": "type: FiberArray\n", + "fiber_arrays/DAS.L001.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", } with pytest.raises(InvalidInventoryError, match="file and a directory"): make_inventory(files) @@ -665,8 +683,8 @@ def test_envelope_spelled_twice(self, make_inventory): """The envelope is one file however it is spelled.""" files = { **MINIMAL, - "inventory.yaml": "type: Inventory\n", - "inventory.json": '{"type": "Inventory"}', + "inventory.yaml": "object_type: Inventory\n", + "inventory.json": '{"object_type": "Inventory"}', } with pytest.raises(InvalidInventoryError, match="more than once"): make_inventory(files) @@ -674,8 +692,10 @@ def test_envelope_spelled_twice(self, make_inventory): def test_two_names_for_one_epoch(self, make_inventory): """Epoch-name uniqueness is temporal rather than textual.""" files = { - "acquisitions/DAS.L001..RAW@2024-06-01.yaml": "type: Acquisition\n", - "acquisitions/DAS.L001..RAW@2024-06-01T000000.yaml": "type: Acquisition\n", + "acquisitions/DAS.L001..RAW@2024-06-01.yaml": "object_type: Acquisition\n", + "acquisitions/DAS.L001..RAW@2024-06-01T000000.yaml": ( + "object_type: Acquisition\n" + ), } with pytest.raises(InvalidInventoryError, match="overlap in time"): make_inventory(files) @@ -684,9 +704,9 @@ def test_overlapping_epochs_of_one_entity(self, make_inventory): """Two epochs of one entity may not overlap, not merely coincide.""" files = { "acquisitions/DAS.L001..RAW@2024-01-01.yaml": ( - "type: Acquisition\nend_time: '2024-08-01'\n" + "object_type: Acquisition\nend_time: '2024-08-01'\n" ), - "acquisitions/DAS.L001..RAW@2024-06-01.yaml": "type: Acquisition\n", + "acquisitions/DAS.L001..RAW@2024-06-01.yaml": "object_type: Acquisition\n", } # Both files are named, since the entity alone would not say which. with pytest.raises(InvalidInventoryError, match="RAW@2024-01-01"): @@ -704,7 +724,7 @@ def test_missing_attrs_file(self, make_inventory): def test_stray_object_file(self, make_inventory): """An entity directory's object file is named attrs.""" - files = {"fiber_arrays/DAS.L001/array.yaml": "type: FiberArray\n"} + files = {"fiber_arrays/DAS.L001/array.yaml": "object_type: FiberArray\n"} with pytest.raises(InvalidInventoryError, match="holds only its attrs"): make_inventory(files) @@ -714,7 +734,7 @@ def test_field_note_beside_an_attrs_file(self, make_inventory): { **MINIMAL, "fiber_arrays/DAS.L001/attrs.yaml": ( - "type: FiberArray\nname: from a directory\n" + "object_type: FiberArray\nname: from a directory\n" ), # Declares nothing, so it participates in nothing -- exactly # as it would one directory deeper. @@ -726,8 +746,8 @@ def test_field_note_beside_an_attrs_file(self, make_inventory): def test_attrs_spelled_twice(self, make_inventory): """One identity is spelled once inside the directory too.""" files = { - "fiber_arrays/DAS.L001/attrs.yaml": "type: FiberArray\n", - "fiber_arrays/DAS.L001/attrs.json": '{"type": "FiberArray"}', + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/attrs.json": '{"object_type": "FiberArray"}', } with pytest.raises(InvalidInventoryError, match="more than once"): make_inventory(files) @@ -751,7 +771,7 @@ def test_hidden_and_foreign_entries_in_an_entity(self, tmp_path): { **MINIMAL, "fiber_arrays/DAS.L001/attrs.yaml": ( - "type: FiberArray\nname: from a directory\n" + "object_type: FiberArray\nname: from a directory\n" ), "fiber_arrays/DAS.L001/photos/wellhead.jpg": "not text", "fiber_arrays/DAS.L001/notes.txt": "a note\n", @@ -776,7 +796,7 @@ class TestSeams: def test_track_table_in_an_entity_directory(self, make_inventory): """A track table is refused by name rather than ignored.""" files = { - "fiber_arrays/DAS.L001/attrs.yaml": "type: FiberArray\n", + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", "fiber_arrays/DAS.L001/coupling.csv": "start_distance\n0\n", } with pytest.raises(InvalidInventoryError, match="track table"): @@ -785,9 +805,9 @@ def test_track_table_in_an_entity_directory(self, make_inventory): def test_optical_path_epoch_directory(self, make_inventory): """An optical path epoch is refused by name rather than ignored.""" files = { - "fiber_arrays/DAS.L001/attrs.yaml": "type: FiberArray\n", + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", "fiber_arrays/DAS.L001/path@2024-05-12T103000/attrs.yaml": ( - "type: OpticalPath\n" + "object_type: OpticalPath\n" ), } with pytest.raises(InvalidInventoryError, match="optical path epoch"): @@ -808,7 +828,9 @@ class TestEpochTimestamps: ) def test_malformed(self, make_inventory, stamp): """A name which claims an epoch and gets it wrong raises.""" - files = {f"acquisitions/DAS.L001..RAW@{stamp}.yaml": "type: Acquisition\n"} + files = { + f"acquisitions/DAS.L001..RAW@{stamp}.yaml": "object_type: Acquisition\n" + } with pytest.raises(InvalidInventoryError, match="epoch timestamp"): make_inventory(files) @@ -817,7 +839,9 @@ def test_malformed(self, make_inventory, stamp): ) def test_timezone_designator(self, make_inventory, stamp): """Naive means UTC, so a designator is refused rather than ignored.""" - files = {f"acquisitions/DAS.L001..RAW@{stamp}.yaml": "type: Acquisition\n"} + files = { + f"acquisitions/DAS.L001..RAW@{stamp}.yaml": "object_type: Acquisition\n" + } with pytest.raises(InvalidInventoryError, match="timezone designator"): make_inventory(files) @@ -825,7 +849,7 @@ def test_negative_offset(self, make_inventory): """An offset in the time portion is a designator, not a malformed time.""" files = { "acquisitions/DAS.L001..RAW@2024-05-12T103000-0600.yaml": ( - "type: Acquisition\n" + "object_type: Acquisition\n" ) } with pytest.raises(InvalidInventoryError, match="timezone designator"): @@ -838,7 +862,9 @@ def test_negative_offset(self, make_inventory): ) def test_outside_the_representable_range(self, make_inventory, stamp): """A nanosecond timestamp wraps silently, so the name is refused.""" - files = {f"acquisitions/DAS.L001..RAW@{stamp}.yaml": "type: Acquisition\n"} + files = { + f"acquisitions/DAS.L001..RAW@{stamp}.yaml": "object_type: Acquisition\n" + } with pytest.raises(InvalidInventoryError, match="outside the range"): make_inventory(files) @@ -847,7 +873,7 @@ def test_the_last_representable_instant(self, make_inventory): out = make_inventory( { "acquisitions/DAS.L001..RAW@2262-04-11T234716.yaml": ( - "type: Acquisition\n" + "object_type: Acquisition\n" ) } ) @@ -858,7 +884,7 @@ def test_two_epoch_markers(self, make_inventory): """A name carries at most one epoch.""" files = { "acquisitions/DAS.L001..RAW@2024-06-01@2024-07-01.yaml": ( - "type: Acquisition\n" + "object_type: Acquisition\n" ) } with pytest.raises(InvalidInventoryError, match="more than one"): @@ -866,13 +892,16 @@ def test_two_epoch_markers(self, make_inventory): def test_empty_epoch(self, make_inventory): """A trailing marker names no epoch.""" - files = {"acquisitions/DAS.L001..RAW@.yaml": "type: Acquisition\n"} + files = {"acquisitions/DAS.L001..RAW@.yaml": "object_type: Acquisition\n"} with pytest.raises(InvalidInventoryError, match="names no epoch"): make_inventory(files) def test_resources_have_no_epochs(self, make_inventory): """A resource is not time-ranged, so its name states no epoch.""" - files = {**MINIMAL, "resources/int_01@2024-06-01.yaml": "type: Interrogator\n"} + files = { + **MINIMAL, + "resources/int_01@2024-06-01.yaml": "object_type: Interrogator\n", + } with pytest.raises(InvalidInventoryError, match="have none"): make_inventory(files) @@ -884,9 +913,9 @@ def test_child_outside_every_epoch(self, make_inventory): """A child falling in no epoch of its container is misfiled.""" files = { "fiber_arrays/DAS.L001.yaml": ( - "type: FiberArray\nend_time: '2024-06-01'\n" + "object_type: FiberArray\nend_time: '2024-06-01'\n" ), - "acquisitions/DAS.L001..RAW@2024-07-01.yaml": "type: Acquisition\n", + "acquisitions/DAS.L001..RAW@2024-07-01.yaml": "object_type: Acquisition\n", } with pytest.raises(InvalidInventoryError, match="0 epochs effective"): make_inventory(files) @@ -895,10 +924,10 @@ def test_ambiguous_child(self, make_inventory): """An unset start beside several container epochs is ambiguous.""" files = { "fiber_arrays/DAS.L001.yaml": ( - "type: FiberArray\nend_time: '2024-06-01'\n" + "object_type: FiberArray\nend_time: '2024-06-01'\n" ), - "fiber_arrays/DAS.L001@2024-06-01.yaml": "type: FiberArray\n", - "acquisitions/DAS.L001..RAW.yaml": "type: Acquisition\n", + "fiber_arrays/DAS.L001@2024-06-01.yaml": "object_type: FiberArray\n", + "acquisitions/DAS.L001..RAW.yaml": "object_type: Acquisition\n", } with pytest.raises(InvalidInventoryError, match="2 epochs effective at any"): make_inventory(files) @@ -907,9 +936,9 @@ def test_station_placed_in_a_network_epoch(self, make_inventory): """Networks epoch like everything else which is time-ranged.""" out = make_inventory( { - "networks/DAS.yaml": "type: Network\nend_time: '2024-06-01'\n", - "networks/DAS@2024-06-01.yaml": "type: Network\nname: later\n", - "stations/DAS.STA1@2024-07-01.yaml": "type: Station\n", + "networks/DAS.yaml": "object_type: Network\nend_time: '2024-06-01'\n", + "networks/DAS@2024-06-01.yaml": "object_type: Network\nname: later\n", + "stations/DAS.STA1@2024-07-01.yaml": "object_type: Station\n", } ) by_name = {x.name: x for x in out.networks} @@ -922,20 +951,23 @@ class TestEnvelope: def test_wrong_type(self, make_inventory): """The envelope declares its type under the same rule as any file.""" - files = {**MINIMAL, "inventory.yaml": "type: Network\n"} + files = {**MINIMAL, "inventory.yaml": "object_type: Network\n"} with pytest.raises(InvalidInventoryError, match="envelope declares"): make_inventory(files) @pytest.mark.parametrize("field", ["networks", "resources"]) def test_collections_are_refused(self, make_inventory, field): """The collections live in the directory structure.""" - files = {**MINIMAL, "inventory.yaml": f"type: Inventory\n{field}: []\n"} + files = {**MINIMAL, "inventory.yaml": f"object_type: Inventory\n{field}: []\n"} with pytest.raises(InvalidInventoryError, match="directory structure"): make_inventory(files) def test_unknown_field_names_the_file(self, make_inventory): """A typo in the envelope says which file states it.""" - files = {**MINIMAL, "inventory.yaml": "type: Inventory\nschema_verison: 1\n"} + files = { + **MINIMAL, + "inventory.yaml": "object_type: Inventory\nschema_verison: 1\n", + } with pytest.raises(InvalidInventoryError, match=r"inventory\.yaml"): make_inventory(files) @@ -971,12 +1003,31 @@ def test_empty_container_is_an_empty_inventory(self, tmp_path): class TestModelAssumptions: """Pin what this loader assumes about the models it builds.""" - def test_only_union_members_carry_a_type_field(self): - """The type tag is a real field only where it discriminates a union.""" + def test_only_union_members_declare_the_tag_as_a_field(self): + """The tag is a real field only where it discriminates a union.""" checked = [] for name, container in loader._CONTAINERS.items(): for model in container.models: - assert ("type" in model.model_fields) == (name == "resources") + assert (TAG_FIELD in model.model_fields) == (name == "resources") + checked.append(model) + assert len(checked) == 9 + + def test_every_model_reads_its_own_tag(self): + """ + The loader leaves the tag in the data for the model to check. + + Before every model read its own, the loader had to pop the tag for + the ones which do not declare it, since they forbid extra fields. + """ + checked = [] + for container in loader._CONTAINERS.values(): + for model in container.models: + # The addressed models are named by their file, so a code + # is the one thing they will not default. + fields = {"code": "X"} if "code" in model.model_fields else {} + assert isinstance(model(**fields, **{TAG_FIELD: model.__name__}), model) + with pytest.raises(ValidationError): + model(**fields, **{TAG_FIELD: "Inventory"}) checked.append(model) assert len(checked) == 9 @@ -991,6 +1042,21 @@ def test_epoch_bearing_models_are_time_ranged(self): checked.append(name) assert len(checked) == len(loader._CONTAINERS) == 5 - def test_inventory_models_forbid_extra_fields(self): - """Type must be popped: an inventory model refuses unknown input.""" + def test_inventory_models_refuse_unknown_input(self): + """ + An inventory model refuses a key it does not declare. + + Which is why the tag has to be consumed rather than ignored, and is + checked on the concrete models rather than on the base, since a + subclass may override the config (PatchAttrs does, in the other + hierarchy). + """ assert InventoryModel.model_config["extra"] == "forbid" + checked = [] + for container in loader._CONTAINERS.values(): + for model in container.models: + fields = {"code": "X"} if "code" in model.model_fields else {} + with pytest.raises(ValidationError, match="not permitted"): + model(**fields, nonsense_key=1) + checked.append(model) + assert len(checked) == 9 diff --git a/tests/test_io/test_dasdae/test_dasdae.py b/tests/test_io/test_dasdae/test_dasdae.py index 533128f98..fd47d041e 100644 --- a/tests/test_io/test_dasdae/test_dasdae.py +++ b/tests/test_io/test_dasdae/test_dasdae.py @@ -21,6 +21,7 @@ from dascore.io.dasdae._compat import translate_legacy_attrs from dascore.io.dasdae.core import DASDAEV1 from dascore.io.dasdae.utils import ( + _ATTRS_CLASS_KEY, _SEPARATE_ATTRS_KEY, _decode_attr_value, _decode_legacy_attr_value, @@ -33,6 +34,7 @@ _save_array, _save_patch, ) +from dascore.io.odh4.core import ODH4PatchAttrs from dascore.utils.downloader import fetch from dascore.utils.misc import register_func from dascore.utils.time import to_datetime64 @@ -337,6 +339,57 @@ def test_get_patch_summary_has_file_metadata(self, random_spool): assert out["source_patch_id"].notnull().all() +class TestAttrsClassRoundTrip: + """A file names the attrs class it holds, so a custom one comes back.""" + + @pytest.fixture(scope="class") + def odh4_patch(self, random_patch): + """A patch carrying a format's own attrs class.""" + attrs = ODH4PatchAttrs(**dict(random_patch.attrs), gauge_length=10.0) + return random_patch.update(attrs=attrs) + + @pytest.fixture(scope="class") + def odh4_path(self, odh4_patch, tmp_path_factory): + """The patch above, written to a DASDAE file.""" + path = tmp_path_factory.mktemp("attrs_class") / "odh4.h5" + odh4_patch.io.write(path, "dasdae") + return path + + def test_read_keeps_the_class(self, odh4_path): + """Reading rebuilds the class which was written.""" + attrs = dc.read(odh4_path)[0].attrs + assert isinstance(attrs, ODH4PatchAttrs) + assert attrs.gauge_length == 10.0 + + def test_scan_keeps_the_class(self, odh4_path): + """Scanning takes the same path, so it says the same thing.""" + assert isinstance(dc.scan(odh4_path)[0].attrs, ODH4PatchAttrs) + + def test_a_file_naming_no_class_reads_as_patch_attrs(self, odh4_path, tmp_path): + """A file written before the class was recorded still reads.""" + path = tmp_path / "unnamed.h5" + shutil.copy(odh4_path, path) + with h5py.File(path, "a") as h5: + for group in h5["waveforms"].values(): + del group.attrs[_ATTRS_CLASS_KEY] + attrs = dc.read(path)[0].attrs + assert type(attrs) is dc.PatchAttrs + # The values were always kept; it is the class which was lost. + assert attrs.gauge_length == 10.0 + + def test_an_unresolvable_class_warns_and_falls_back(self, odh4_path, tmp_path): + """An archive written by a package we lack is still readable.""" + path = tmp_path / "foreign.h5" + shutil.copy(odh4_path, path) + with h5py.File(path, "a") as h5: + for group in h5["waveforms"].values(): + group.attrs[_ATTRS_CLASS_KEY] = "absent:ForeignAttrs" + with pytest.warns(UserWarning, match="Nothing registers"): + attrs = dc.read(path)[0].attrs + assert type(attrs) is dc.PatchAttrs + assert attrs.gauge_length == 10.0 + + class TestLegacyFixtureCompatibility: """Tests for the retained legacy DASDAE fixture compatibility helpers.""" diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 000000000..c150471be --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,613 @@ +"""Tests for DASCore models and related functionality.""" + +from __future__ import annotations + +import json +import pickle +import subprocess +import sys + +import numpy as np +import pydantic +import pytest +from pydantic import Field, ValidationError + +import dascore.models as dc_models +from dascore.core.attrs import PatchAttrs +from dascore.core.inventory import Cable, Inventory +from dascore.exceptions import InvalidModelTagError +from dascore.io.core import FiberIO +from dascore.io.sintela.core import SintelaPatchAttrs +from dascore.models import ( + DascoreBaseModel, + DateTime64, + FrozenDictType, + OptionalFiniteFloat, + TimeDelta64, + UnitQuantity, + registry, + sensible_model_equals, +) +from dascore.units import Quantity +from dascore.utils import models as models_shim + + +class _TestModel(DascoreBaseModel): + array: np.ndarray | None = None + _private: int = 0 + some_str: str = "10" + + +class _Inner(DascoreBaseModel): + """A nested model, which a document must also name.""" + + value: int = 1 + + +class _Outer(DascoreBaseModel): + """A model holding another.""" + + inner: _Inner = Field(default_factory=_Inner) + + +class TestModelEquals: + """Tests for seeing if models/dicts are equal.""" + + def test_empty(self): + """Empty dicts should be equal.""" + assert sensible_model_equals({}, {}) + + def test_arrays_not_equal(self): + """Ensure when arrays aren't equal models aren't.""" + mod1 = _TestModel(array=np.arange(10)) + mod2 = _TestModel(array=np.arange(10) + 10) + assert not sensible_model_equals(mod1, mod2) + + def test_private(self): + """When private attrs aren't equal the models should still be.""" + mod1 = _TestModel(_private=1) + mod2 = _TestModel(_private=2) + assert sensible_model_equals(mod1, mod2) + + def test_private_disjoint(self): + """Private attrs not shared should not affect equality.""" + mod1 = _TestModel(_private_1=1) + mod2 = _TestModel(_private_2=2) + assert sensible_model_equals(mod1, mod2) + + def test_new(self): + """Ensure a new model can b e created.""" + mod = _TestModel(some_str="test") + new = mod.new(some_str="bob") + assert new.some_str == "bob" + assert new is not mod + + +class _UnitModel(DascoreBaseModel): + """A model carrying units.""" + + units: UnitQuantity | None = None + + +class TestUnitQuantity: + """Units name a single unit, so they stay scalar.""" + + @pytest.mark.parametrize( + "spec", ["m/s", "1/s", "Hz", "2*degC", "2.5*degC", "10 m/s", 1.0, None, ""] + ) + def test_real_spellings_accepted(self, spec): + """Every real unit spelling has a scalar magnitude, so it hashes.""" + assert isinstance(hash(_UnitModel(units=spec)), int) + + def test_sequence_refused(self): + """A sequence makes pint build a mutable, unhashable magnitude.""" + with pytest.raises(ValidationError, match="single unit"): + _UnitModel(units=(1.0, 2.0)) + + @pytest.mark.parametrize("magnitude", [np.array([1.0, 2.0]), np.array(1.0)]) + def test_array_magnitude_refused(self, magnitude): + """A quantity built by hand is held to the same rule.""" + # A zero dimensional array is as writable as any other, so measuring + # how many dimensions it has would let this one through. + with pytest.raises(ValidationError, match="single unit"): + _UnitModel(units=Quantity(magnitude, "m")) + + def test_serialized_units_validate_again(self): + """What the serializer writes has to survive being read back.""" + model = _UnitModel(units="m/s") + assert _UnitModel(**model.model_dump()) == model + + +class _NullModel(DascoreBaseModel): + """A model holding every flavor of null a field can carry.""" + + time: DateTime64 = np.datetime64("NaT", "ns") + duration: TimeDelta64 = np.timedelta64("NaT", "ns") + number: float = np.nan + mapping: FrozenDictType[str, float] = Field(default_factory=dict) + + +class TestModelHash: + """Hashing agrees with equality, so models work as dict keys.""" + + def test_unset_models_hash_equally(self): + """Nulls compare equal to nulls, so they must hash alike.""" + # Each null is a fresh object, and nan and NaT both hash by identity. + first, second = _NullModel(), _NullModel() + assert first == second + assert hash(first) == hash(second) + assert {first: "found"}[second] == "found" + + def test_null_from_any_input_hashes_alike(self): + """None and the string 'NaT' reach the same null.""" + assert hash(_NullModel(time=None)) == hash(_NullModel(time="NaT")) + + def test_nulls_inside_a_mapping_hash_alike(self): + """A null nested in a frozen mapping is normalized too.""" + first = _NullModel(mapping={"gain": float("nan")}) + second = _NullModel(mapping={"gain": float("nan")}) + assert first == second + assert hash(first) == hash(second) + + def test_mapping_order_does_not_change_the_hash(self): + """Mappings compare without regard to order, so they hash that way.""" + first = _NullModel(mapping={"a": 1.0, "b": 2.0}) + second = _NullModel(mapping={"b": 2.0, "a": 1.0}) + assert first == second + assert hash(first) == hash(second) + + def test_hash_survives_a_pickle_round_trip(self): + """Unpickling skips validation, so the hash cannot depend on it.""" + model = _NullModel() + loaded = pickle.loads(pickle.dumps(model)) + assert loaded == model + assert hash(loaded) == hash(model) + + def test_differing_models_hash_apart(self): + """A real value is not confused with the null it replaces.""" + assert hash(_NullModel(number=1.0)) != hash(_NullModel()) + + def test_declaration_order_does_not_change_the_hash(self): + """Equality compares field names, so declaration order cannot matter.""" + + class Forward(DascoreBaseModel): + a: int = 1 + b: int = 2 + + class Reversed(DascoreBaseModel): + b: int = 2 + a: int = 1 + + first, second = Forward(), Reversed() + assert first == second + assert hash(first) == hash(second) + + def test_unhashable_field_still_refuses(self): + """A model holding an array is unhashable, and says so.""" + with pytest.raises(TypeError, match="unhashable"): + hash(_TestModel(array=np.arange(10))) + + +# A floor, not a count: raise it when a format adds a class. It exists +# because FiberIO swallows a plugin's import failure as a warning, which +# would otherwise drop that format's class out of the walk below and leave +# this file green while testing one format fewer. +_ATTRS_CLASS_FLOOR = 18 + + +def _dascore_patch_attrs_classes(): + """Every PatchAttrs class DASCore itself declares, base included.""" + # The subclasses only exist once their io modules are imported, and other + # test modules register their own subclasses globally on import, so the + # walk both forces the load and keeps to DASCore's own classes. + FiberIO.manager.load_plugins() + found: dict[str, type[PatchAttrs]] = {} + stack = [PatchAttrs] + while stack: + cls = stack.pop() + stack.extend(cls.__subclasses__()) + if cls.__module__.startswith("dascore."): + found[cls.__name__] = cls + assert len(found) >= _ATTRS_CLASS_FLOOR, ( + f"only {len(found)} PatchAttrs classes were found; a format's plugin " + "probably failed to import, which would silently go untested here." + ) + return [found[name] for name in sorted(found)] + + +# A required field has no default to round trip, so the walk states one. A +# new required field fails the assert below rather than dropping out of it. +_REQUIRED_ATTR_VALUES = {"gauge_length": 10.0} + + +def _optional_float_names(cls): + """Names of the fields on a class which hold an optional number.""" + annotation = OptionalFiniteFloat.__origin__ + return [n for n, f in cls.model_fields.items() if f.annotation == annotation] + + +def _minimal_attrs(cls): + """Build the emptiest legal instance of a PatchAttrs class.""" + required = {name for name, f in cls.model_fields.items() if f.is_required()} + assert not (unknown := required - set(_REQUIRED_ATTR_VALUES)), ( + f"{cls.__name__} requires {sorted(unknown)}, which " + "_REQUIRED_ATTR_VALUES does not state a value for." + ) + return cls(**{name: _REQUIRED_ATTR_VALUES[name] for name in required}) + + +@pytest.mark.parametrize( + "attrs_class", + _dascore_patch_attrs_classes(), + ids=lambda cls: cls.__name__, +) +class TestPatchAttrsSerialization: + """ + Every PatchAttrs class must survive a text round trip. + + Parametrized over the class walk rather than a list, so a format added + later is covered without touching this file. + """ + + def test_json_round_trip(self, attrs_class): + """A defaulted instance reconstructs from its own json.""" + attrs = _minimal_attrs(attrs_class) + out = attrs_class.model_validate_json(attrs.model_dump_json()) + # Compared as text, not with ==, which counts any null equal to any + # other and so cannot see a value degrade to None on the way through. + assert out.model_dump_json() == attrs.model_dump_json() + + def test_no_field_holds_a_number_json_cannot_spell(self, attrs_class): + """ + No field holds a number json cannot spell. + + NaN and inf write as `null` and then refuse to read back, so an + optional number is spelled `OptionalFiniteFloat`. Checked on the + built instance rather than the raw default, since `np.float32("nan")` + is not a `float` and would slip past that. + """ + instance = _minimal_attrs(attrs_class) + for name in attrs_class.model_fields: + value = getattr(instance, name, None) + if isinstance(value, float | np.floating): + assert np.isfinite(value), ( + f"{attrs_class.__name__}.{name} holds {value}." + ) + + def test_a_non_finite_value_from_a_file_reads_as_absent(self, attrs_class): + """ + A format which spells "unknown" as nan is read, not refused. + + Readers hand vendor header floats straight to these classes, and a + scan swallows a ValidationError as "failed to scan", so refusing one + would silently drop the file from a spool rather than report it. + """ + for name in _optional_float_names(attrs_class): + built = _minimal_attrs(attrs_class).new(**{name: np.float32("nan")}) + assert getattr(built, name) is None + + +@pytest.fixture +def clean_registry(): + """Undo whatever a test registers, so the real registry is untouched.""" + before = registry.registered_models() + yield + registry._REGISTRY.clear() + registry._REGISTRY.update(before) + + +def _model_in(module: str, name: str = "Square", base=DascoreBaseModel): + """Declare a model as though it lived in another package.""" + return type(name, (base,), {"__module__": module, "__qualname__": name}) + + +class TestOptionalFiniteFloat: + """How a number which may be absent is spelled.""" + + class _Model(DascoreBaseModel): + value: OptionalFiniteFloat = None + + @pytest.mark.parametrize("value", [np.nan, np.inf, -np.inf, np.float32("nan")]) + def test_a_non_finite_number_is_absent(self, value): + """A file which spells "unknown" as nan is read, not refused.""" + assert self._Model(value=value).value is None + + def test_a_number_survives(self): + """Everything else is the number it was.""" + assert self._Model(value=1.5).value == 1.5 + + def test_a_non_number_is_left_to_the_field(self): + """ + Only numbers can be finite, so anything else passes through here. + + A string is what a yaml or json document holds, and pydantic's own + coercion is what should read it -- or refuse it, in its own words. + """ + assert self._Model(value="1.5").value == 1.5 + with pytest.raises(ValidationError): + self._Model(value="not a number") + + +def test_optional_numbers_are_declared_across_the_formats(): + """ + The nan-to-None loop above tests nothing on a class with no such field. + + So the fields it walks are counted here: were the annotation renamed or + the migration reverted, every one of those loops would quietly empty. + """ + total = sum( + len(_optional_float_names(cls)) for cls in _dascore_patch_attrs_classes() + ) + assert total >= 25 + + +class TestDeprecatedModelPath: + """`dascore.utils.models` keeps working for readers which import it.""" + + def test_names_resolve_to_their_new_home(self): + """The same objects, not copies of them.""" + assert models_shim.DascoreBaseModel is DascoreBaseModel + assert models_shim.DateTime64 is DateTime64 + + def test_it_re_exports_everything_the_package_does(self): + """ + Re-exported wholesale, so a name added later cannot go missing. + + Which is the point of the star import: a second hand-maintained + list would drift from the first without anything noticing. + """ + assert set(dc_models.__all__) <= set(models_shim.__all__) + for name in models_shim.__all__: + assert hasattr(models_shim, name), name + + def test_it_still_carries_pydantic_base_model(self): + """It re-exported this, so something out of tree may import it.""" + assert models_shim.BaseModel is pydantic.BaseModel + + +class TestModelTagRegistry: + """A tag names one class, and the registry is what resolves it.""" + + def test_dascore_models_register_bare(self): + """A bare name means dascore, which keeps files hand-authorable.""" + assert registry.registered_models()["PatchAttrs"] is PatchAttrs + assert registry.get_model_tag(PatchAttrs) == "PatchAttrs" + + def test_out_of_tree_models_are_namespaced(self, clean_registry): + """A plugin's namespace is derived, so it needs no ceremony.""" + cls = _model_in("myplugin.shapes") + assert registry.get_model_tag(cls) == "myplugin:Square" + assert registry.registered_models()["myplugin:Square"] is cls + + def test_colliding_dascore_names_raise(self, clean_registry): + """Two of our own classes may not claim one tag; this is the pin.""" + first = _model_in("dascore.somewhere", "Doubled") + with pytest.raises(InvalidModelTagError, match="claim the tag"): + _model_in("dascore.elsewhere", "Doubled") + # The collision does not replace what was there before it. + assert registry.registered_models()["Doubled"] is first + + def test_colliding_plugin_names_stop_resolving(self, clean_registry): + """ + A user cannot rename another package's class, so importing both works. + + What the tag may not do is quietly pick one: a document written by + the first would then be read as the second. + """ + _model_in("myplugin.a", "Doubled") + with pytest.warns(UserWarning, match="claim the tag"): + _model_in("myplugin.b", "Doubled") + assert "myplugin:Doubled" not in registry.registered_models() + with pytest.raises(InvalidModelTagError, match="names two classes"): + registry.resolve_model_tag("myplugin:Doubled") + + def test_an_uppercase_package_is_a_legal_namespace(self, clean_registry): + """ + A package name may start with a capital, and many do (PIL, Terra15). + + The tag DASCore writes must be one it can read: a namespace the + grammar refused would make its own files unreadable. + """ + cls = _model_in("Terra15.attrs", "Terra15Attrs") + tag = registry.get_model_tag(cls) + assert tag == "Terra15:Terra15Attrs" + assert registry.resolve_model_tag(tag) is cls + + def test_a_class_which_cannot_be_named_is_not_registered(self, clean_registry): + """ + A parametrized generic is spelled `G[int]`, which no document states. + + Writing that tag anyway would produce a file DASCore refuses to + read, so such a class is simply not named. + """ + cls = _model_in("myplugin.generics", "Square[int]") + assert registry.get_model_tag(cls) is None + assert "myplugin:Square[int]" not in registry.registered_models() + + def test_a_class_declared_in_a_function_is_not_registered(self, clean_registry): + """Nothing can resolve a name which exists only while a call runs.""" + + class Local(DascoreBaseModel): + """A model which cannot be addressed from a document.""" + + assert "Local" not in registry.registered_models() + + def test_a_reimported_module_replaces_its_own_entry(self, clean_registry): + """Re-importing a module is not two classes claiming one tag.""" + _model_in("myplugin.shapes") + second = _model_in("myplugin.shapes") + assert registry.registered_models()["myplugin:Square"] is second + + @pytest.mark.parametrize( + "tag", ["Cable", "myplugin:Square", "my_plugin.sub:Square", "Cable-0.0.1"] + ) + def test_legal_tags(self, tag): + """The grammar takes a name, a namespace and room for a version.""" + # Through the function rather than the pattern: what matters is that + # a legal tag is looked up rather than refused outright. + registry.resolve_model_tag(tag) + + @pytest.mark.parametrize( + "tag", ["", "9Cable", "Cable-1", ":Cable", "dascore.core.inventory.Cable", 3] + ) + def test_illegal_tags_are_refused(self, tag): + """A tag which is not a tag is a malformed document, not an unknown one.""" + with pytest.raises(InvalidModelTagError, match="is not a legal"): + registry.resolve_model_tag(tag) + + def test_an_unknown_tag_falls_back_with_a_warning(self): + """A document from an uninstalled package still reads as its base.""" + with pytest.warns(UserWarning, match="Nothing registers"): + out = registry.resolve_tagged_model("absent:Whatever", default=PatchAttrs) + assert out is PatchAttrs + + def test_a_resolved_tag_must_be_the_kind_asked_for(self): + """A file naming a class of the wrong kind is refused, not built.""" + with pytest.raises(InvalidModelTagError, match="not a PatchAttrs"): + registry.resolve_tagged_model("Cable", default=PatchAttrs) + + def test_an_unknown_tag_without_a_default_raises(self): + """A standalone document has no other class to fall back on.""" + with pytest.raises(InvalidModelTagError, match="Nothing registers"): + registry.resolve_tagged_model("absent:Whatever") + + def test_an_untagged_document_without_a_default_raises(self): + """Nothing but the document says what a standalone document holds.""" + with pytest.raises(InvalidModelTagError, match="declares no"): + registry.resolve_tagged_model(None) + + def test_an_untagged_document_takes_the_default(self): + """A caller which names the class does not need the document to.""" + assert registry.resolve_tagged_model(None, default=PatchAttrs) is PatchAttrs + + def test_a_broken_plugin_does_not_break_resolution(self, monkeypatch): + """A plugin which cannot be imported has no models to find.""" + + def _boom(): + raise ImportError("this plugin is a stale entry point") + + monkeypatch.setattr(registry, "_plugins_swept", False) + monkeypatch.setattr( + registry, "get_entry_point_loaders", lambda group: {"broken": _boom} + ) + assert registry.resolve_model_tag("absent:Whatever") is None + + @pytest.mark.concurrency + def test_a_model_in_an_unimported_module_is_found(self): + """ + A format's models only exist once its module is imported. + + Run in a fresh interpreter because the io modules are imported by + the time any other test runs, which is exactly what hides this. + """ + code = ( + "import dascore.models.registry as r\n" + "assert 'ODH4PatchAttrs' not in r.registered_models(), 'already there'\n" + "assert r.resolve_model_tag('ODH4PatchAttrs') is not None, 'not found'\n" + ) + out = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True + ) + assert out.returncode == 0, out.stderr + + +class TestObjectTypeSerialization: + """Every model names its class in a text document, and nowhere else.""" + + def test_json_states_the_class(self): + """A document says what it holds.""" + dumped = json.loads(PatchAttrs(tag="a").model_dump_json()) + assert dumped[registry.TAG_FIELD] == "PatchAttrs" + + def test_a_python_dump_is_untagged(self): + """ + The tag is not a field, and python dumps are not documents. + + Equality compares them, `new` reconstructs from them and the spool + index ingests them; a key which is not a field belongs in none. + """ + assert registry.TAG_FIELD not in PatchAttrs().model_dump() + assert registry.TAG_FIELD not in PatchAttrs().new(tag="a").model_dump() + + def test_include_and_exclude_still_filter_fields(self): + """ + A model serializer must not cost the caller `include`/`exclude`. + + Declaring it with `when_used="json"` reads better and silently + breaks them: pydantic skips the wrapper in python mode, and the + field filtering goes with it. `Patch.equals` dumps with `include`. + """ + attrs = PatchAttrs(tag="a") + assert set(attrs.model_dump(include={"tag"})) == {"tag"} + assert "tag" not in attrs.model_dump(exclude={"tag"}) + + def test_nested_models_state_their_class(self): + """Universal, so a nested object can be read on its own later.""" + dumped = json.loads(_Outer(inner=_Inner()).model_dump_json()) + assert dumped[registry.TAG_FIELD] == "tests:_Outer" + assert dumped["inner"][registry.TAG_FIELD] == "tests:_Inner" + + def test_a_subclass_reads_back_through_its_base(self): + """The document holds everything the base declares, so this is fine.""" + attrs = SintelaPatchAttrs(gauge_length=10.0) + out = PatchAttrs.model_validate_json(attrs.model_dump_json()) + assert out.gauge_length == 10.0 + + def test_a_foreign_tag_is_refused(self): + """A document which names another class is misfiled, not reinterpreted.""" + data = {registry.TAG_FIELD: "Cable"} + with pytest.raises(ValidationError, match="cannot be read as"): + PatchAttrs(**data) + + def test_an_unknown_tag_is_accepted(self): + """The caller named the class, so there is nothing to disagree with.""" + attrs = PatchAttrs(**{registry.TAG_FIELD: "absent:Whatever"}) + assert isinstance(attrs, PatchAttrs) + + @pytest.mark.parametrize("value", ["my_sensor", "a b c", 3]) + def test_an_attr_which_names_no_class_is_kept(self, value): + """ + A reader's own metadata spelled like the tag is data, not a tag. + + PatchAttrs keeps extra fields, so consuming a value which names no + class would lose it silently; a tag DASCore wrote always resolves. + """ + attrs = PatchAttrs(**{registry.TAG_FIELD: value}) + assert attrs.model_dump()[registry.TAG_FIELD] == value + + def test_the_tag_does_not_become_an_extra_field(self): + """PatchAttrs keeps extras, and the tag is not one of them.""" + attrs = PatchAttrs(**{registry.TAG_FIELD: "PatchAttrs"}) + assert not hasattr(attrs, registry.TAG_FIELD) + + def test_a_union_member_writes_its_own_tag(self, clean_registry): + """ + The nine union members declare the tag as a real field. + + Pydantic must pick a class before an object exists, so their tag + cannot be a serializer concern. The base class must leave the value + alone rather than overwrite it with the registry's name for the + class: the two differ for a subclass declared out of tree, and the + closed Literal would refuse to read the registry's spelling back. + """ + dumped = json.loads(Cable(resource_id="c1").model_dump_json()) + assert dumped[registry.TAG_FIELD] == "Cable" + # A plugin's subclass of a union member: tag and Literal disagree. + plugin_cable = type("PluginCable", (Cable,), {"__module__": "myplugin.cables"}) + assert registry.get_model_tag(plugin_cable) == "myplugin:PluginCable" + sub_dump = json.loads(plugin_cable(resource_id="c1").model_dump_json()) + assert sub_dump[registry.TAG_FIELD] == "Cable" + # Which is what keeps the document readable at all. + assert isinstance(Cable(**sub_dump), Cable) + + def test_a_union_member_states_its_tag_under_exclude_defaults(self): + """ + The tag is defaulted, so a dump flag would otherwise drop it. + + The serializer puts it back: a document which cannot say which union + member it holds cannot be read at all. + """ + inventory = Inventory(resources={"c1": Cable(resource_id="c1")}) + dumped = inventory.model_dump(mode="json", exclude_defaults=True) + assert dumped["resources"]["c1"][registry.TAG_FIELD] == "Cable" + assert Inventory(**dumped) == inventory diff --git a/tests/test_utils/test_models.py b/tests/test_utils/test_models.py deleted file mode 100644 index d7a13da0c..000000000 --- a/tests/test_utils/test_models.py +++ /dev/null @@ -1,163 +0,0 @@ -"""Tests for DASCore models and related functionality.""" - -from __future__ import annotations - -import pickle - -import numpy as np -import pytest -from pydantic import Field, ValidationError - -from dascore.units import Quantity -from dascore.utils.models import ( - DascoreBaseModel, - DateTime64, - FrozenDictType, - TimeDelta64, - UnitQuantity, - sensible_model_equals, -) - - -class _TestModel(DascoreBaseModel): - array: np.ndarray | None = None - _private: int = 0 - some_str: str = "10" - - -class TestModelEquals: - """Tests for seeing if models/dicts are equal.""" - - def test_empty(self): - """Empty dicts should be equal.""" - assert sensible_model_equals({}, {}) - - def test_arrays_not_equal(self): - """Ensure when arrays aren't equal models aren't.""" - mod1 = _TestModel(array=np.arange(10)) - mod2 = _TestModel(array=np.arange(10) + 10) - assert not sensible_model_equals(mod1, mod2) - - def test_private(self): - """When private attrs aren't equal the models should still be.""" - mod1 = _TestModel(_private=1) - mod2 = _TestModel(_private=2) - assert sensible_model_equals(mod1, mod2) - - def test_private_disjoint(self): - """Private attrs not shared should not affect equality.""" - mod1 = _TestModel(_private_1=1) - mod2 = _TestModel(_private_2=2) - assert sensible_model_equals(mod1, mod2) - - def test_new(self): - """Ensure a new model can b e created.""" - mod = _TestModel(some_str="test") - new = mod.new(some_str="bob") - assert new.some_str == "bob" - assert new is not mod - - -class _UnitModel(DascoreBaseModel): - """A model carrying units.""" - - units: UnitQuantity | None = None - - -class TestUnitQuantity: - """Units name a single unit, so they stay scalar.""" - - @pytest.mark.parametrize( - "spec", ["m/s", "1/s", "Hz", "2*degC", "2.5*degC", "10 m/s", 1.0, None, ""] - ) - def test_real_spellings_accepted(self, spec): - """Every real unit spelling has a scalar magnitude, so it hashes.""" - assert isinstance(hash(_UnitModel(units=spec)), int) - - def test_sequence_refused(self): - """A sequence makes pint build a mutable, unhashable magnitude.""" - with pytest.raises(ValidationError, match="single unit"): - _UnitModel(units=(1.0, 2.0)) - - @pytest.mark.parametrize("magnitude", [np.array([1.0, 2.0]), np.array(1.0)]) - def test_array_magnitude_refused(self, magnitude): - """A quantity built by hand is held to the same rule.""" - # A zero dimensional array is as writable as any other, so measuring - # how many dimensions it has would let this one through. - with pytest.raises(ValidationError, match="single unit"): - _UnitModel(units=Quantity(magnitude, "m")) - - def test_serialized_units_validate_again(self): - """What the serializer writes has to survive being read back.""" - model = _UnitModel(units="m/s") - assert _UnitModel(**model.model_dump()) == model - - -class _NullModel(DascoreBaseModel): - """A model holding every flavor of null a field can carry.""" - - time: DateTime64 = np.datetime64("NaT", "ns") - duration: TimeDelta64 = np.timedelta64("NaT", "ns") - number: float = np.nan - mapping: FrozenDictType[str, float] = Field(default_factory=dict) - - -class TestModelHash: - """Hashing agrees with equality, so models work as dict keys.""" - - def test_unset_models_hash_equally(self): - """Nulls compare equal to nulls, so they must hash alike.""" - # Each null is a fresh object, and nan and NaT both hash by identity. - first, second = _NullModel(), _NullModel() - assert first == second - assert hash(first) == hash(second) - assert {first: "found"}[second] == "found" - - def test_null_from_any_input_hashes_alike(self): - """None and the string 'NaT' reach the same null.""" - assert hash(_NullModel(time=None)) == hash(_NullModel(time="NaT")) - - def test_nulls_inside_a_mapping_hash_alike(self): - """A null nested in a frozen mapping is normalized too.""" - first = _NullModel(mapping={"gain": float("nan")}) - second = _NullModel(mapping={"gain": float("nan")}) - assert first == second - assert hash(first) == hash(second) - - def test_mapping_order_does_not_change_the_hash(self): - """Mappings compare without regard to order, so they hash that way.""" - first = _NullModel(mapping={"a": 1.0, "b": 2.0}) - second = _NullModel(mapping={"b": 2.0, "a": 1.0}) - assert first == second - assert hash(first) == hash(second) - - def test_hash_survives_a_pickle_round_trip(self): - """Unpickling skips validation, so the hash cannot depend on it.""" - model = _NullModel() - loaded = pickle.loads(pickle.dumps(model)) - assert loaded == model - assert hash(loaded) == hash(model) - - def test_differing_models_hash_apart(self): - """A real value is not confused with the null it replaces.""" - assert hash(_NullModel(number=1.0)) != hash(_NullModel()) - - def test_declaration_order_does_not_change_the_hash(self): - """Equality compares field names, so declaration order cannot matter.""" - - class Forward(DascoreBaseModel): - a: int = 1 - b: int = 2 - - class Reversed(DascoreBaseModel): - b: int = 2 - a: int = 1 - - first, second = Forward(), Reversed() - assert first == second - assert hash(first) == hash(second) - - def test_unhashable_field_still_refuses(self): - """A model holding an array is unhashable, and says so.""" - with pytest.raises(TypeError, match="unhashable"): - hash(_TestModel(array=np.arange(10)))