From c52560795b57e5a0dc2b53ecf5eab005674b8217 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Thu, 13 Aug 2026 12:16:46 +0200 Subject: [PATCH 1/8] Promote the model layer to dascore/models utils/models.py held annotated types, the base model and the inventory bases; a registry and a tag serializer were about to join them. Split into a package and leave the old path re-exporting, since out-of-tree readers import their types from it. --- dascore/core/attrs.py | 2 +- dascore/core/coordmanager.py | 12 +- dascore/core/coords.py | 10 +- dascore/core/inventory.py | 14 +- dascore/core/inventory_loader.py | 2 +- dascore/core/patch.py | 2 +- dascore/core/summary.py | 2 +- dascore/io/ai4eps/core.py | 2 +- dascore/io/febus/core.py | 2 +- dascore/io/optodas/core.py | 2 +- dascore/io/prodml/utils.py | 2 +- dascore/io/sintela/protobuf_utils.py | 2 +- dascore/io/utils.py | 2 +- dascore/io/xml_binary/core.py | 2 +- dascore/io/xml_binary/utils.py | 4 +- dascore/models/__init__.py | 53 ++++ dascore/models/base.py | 221 +++++++++++++++ dascore/models/types.py | 99 +++++++ dascore/proc/basic.py | 2 +- dascore/proc/inventory.py | 2 +- dascore/proc/mute.py | 2 +- dascore/utils/array.py | 2 +- dascore/utils/coordmanager.py | 2 +- dascore/utils/models.py | 346 ++++------------------- tests/test_core/test_inventory.py | 2 +- tests/test_core/test_inventory_loader.py | 2 +- tests/{test_utils => }/test_models.py | 4 +- 27 files changed, 461 insertions(+), 338 deletions(-) create mode 100644 dascore/models/__init__.py create mode 100644 dascore/models/base.py create mode 100644 dascore/models/types.py rename tests/{test_utils => }/test_models.py (99%) diff --git a/dascore/core/attrs.py b/dascore/core/attrs.py index e8ca3ab15..2f276e92f 100644 --- a/dascore/core/attrs.py +++ b/dascore/core/attrs.py @@ -19,11 +19,11 @@ DataType, max_lens, ) +from dascore.models import DascoreBaseModel, UnitQuantity from dascore.utils.misc import ( to_str, validate_acquisition_key, ) -from dascore.utils.models import DascoreBaseModel, UnitQuantity str_validator = PlainValidator(to_str) diff --git a/dascore/core/coordmanager.py b/dascore/core/coordmanager.py index f52873f6b..c61ccea9f 100644 --- a/dascore/core/coordmanager.py +++ b/dascore/core/coordmanager.py @@ -66,6 +66,12 @@ ParameterError, PatchBroadcastError, ) +from dascore.models import ( + ArrayLike, + DascoreBaseModel, + frozen_dict_serializer, + frozen_dict_validator, +) from dascore.utils.docs import compose_docstring from dascore.utils.mapping import FrozenDict from dascore.utils.misc import ( @@ -75,12 +81,6 @@ cached_method, iterate, ) -from dascore.utils.models import ( - ArrayLike, - DascoreBaseModel, - frozen_dict_serializer, - frozen_dict_validator, -) MaybeArray = ArrayLike | np.ndarray | None diff --git a/dascore/core/coords.py b/dascore/core/coords.py index f2a0d7435..a0cf22c17 100644 --- a/dascore/core/coords.py +++ b/dascore/core/coords.py @@ -35,6 +35,11 @@ from dascore.compat import array, is_array from dascore.constants import _AGG_FUNCS, DIM_REDUCE_DOCS, dascore_styles from dascore.exceptions import CoordError, ParameterError +from dascore.models import ( + ArrayLike, + DascoreBaseModel, + UnitQuantity, +) from dascore.units import ( Quantity, Unit, @@ -63,11 +68,6 @@ iterate, sanitize_range_param, ) -from dascore.utils.models import ( - ArrayLike, - DascoreBaseModel, - UnitQuantity, -) from dascore.utils.time import ( dtype_time_like, is_datetime64, diff --git a/dascore/core/inventory.py b/dascore/core/inventory.py index 47fae6b60..907798b4b 100644 --- a/dascore/core/inventory.py +++ b/dascore/core/inventory.py @@ -44,6 +44,13 @@ from dascore.constants import DataCategory, DataType from dascore.exceptions import InvalidInventoryError, ParameterError +from dascore.models import ( + DateTime64, + FrozenDictType, + InventoryModel, + TimeRangedModel, + UnitQuantity, +) from dascore.utils.mapping import FrozenDict from dascore.utils.misc import ( check_code, @@ -51,13 +58,6 @@ optional_import, validate_acquisition_key, ) -from dascore.utils.models import ( - DateTime64, - FrozenDictType, - InventoryModel, - TimeRangedModel, - UnitQuantity, -) CouplingType = Literal[ "conduit", diff --git a/dascore/core/inventory_loader.py b/dascore/core/inventory_loader.py index 9c023f0d5..b61040207 100644 --- a/dascore/core/inventory_loader.py +++ b/dascore/core/inventory_loader.py @@ -56,8 +56,8 @@ InvalidInventoryError, MissingOptionalDependencyError, ) +from dascore.models import InventoryModel, TimeRangedModel from dascore.utils.misc import check_code, optional_import -from dascore.utils.models import InventoryModel, TimeRangedModel from dascore.utils.time import to_datetime64 # One data model stands behind all three spellings, so they are accepted diff --git a/dascore/core/patch.py b/dascore/core/patch.py index 1ccc8f3ca..81253d4d2 100644 --- a/dascore/core/patch.py +++ b/dascore/core/patch.py @@ -18,6 +18,7 @@ from dascore.core.attrs import PatchAttrs from dascore.core.coordmanager import CoordManager, get_coord_manager from dascore.core.summary import PatchSummary +from dascore.models import ArrayLike from dascore.utils.array import ( PatchUFunc, apply_ufunc, @@ -25,7 +26,6 @@ patch_array_ufunc, ) from dascore.utils.display import array_to_text, attrs_to_text, get_dascore_text -from dascore.utils.models import ArrayLike from dascore.utils.namespace import NamespaceOwner from dascore.utils.patch import check_patch_attrs, check_patch_coords, get_patch_names from dascore.utils.time import to_float diff --git a/dascore/core/summary.py b/dascore/core/summary.py index ba3ab5dec..72749c084 100644 --- a/dascore/core/summary.py +++ b/dascore/core/summary.py @@ -17,7 +17,7 @@ from dascore.constants import path_types from dascore.core.attrs import PatchAttrs from dascore.core.coords import CoordSummary -from dascore.utils.models import DascoreBaseModel +from dascore.models import DascoreBaseModel from dascore.utils.paths import coerce_to_upath, is_pathlike diff --git a/dascore/io/ai4eps/core.py b/dascore/io/ai4eps/core.py index b41ab0cd6..364fb6561 100644 --- a/dascore/io/ai4eps/core.py +++ b/dascore/io/ai4eps/core.py @@ -9,8 +9,8 @@ import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload +from dascore.models import DateTime64 from dascore.utils.hdf5 import H5Reader -from dascore.utils.models import DateTime64 from .utils import _get_attrs_dict, _get_coords, _get_patches, _is_ai4eps diff --git a/dascore/io/febus/core.py b/dascore/io/febus/core.py index fac9dd08e..a9fe973f1 100644 --- a/dascore/io/febus/core.py +++ b/dascore/io/febus/core.py @@ -18,9 +18,9 @@ ) from dascore.io import FiberIO, ScanPayload from dascore.io.core import make_scan_payload +from dascore.models import UTF8Str from dascore.utils.hdf5 import H5Reader from dascore.utils.io import TextReader -from dascore.utils.models import UTF8Str from .a1utils import ( _get_febus_version_str, diff --git a/dascore/io/optodas/core.py b/dascore/io/optodas/core.py index f95e323ea..0797c2f9d 100644 --- a/dascore/io/optodas/core.py +++ b/dascore/io/optodas/core.py @@ -9,8 +9,8 @@ import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload +from dascore.models import UTF8Str from dascore.utils.hdf5 import H5Reader -from dascore.utils.models import UTF8Str from .utils import _get_opto_das_attrs, _get_opto_das_version_str, _read_opto_das diff --git a/dascore/io/prodml/utils.py b/dascore/io/prodml/utils.py index 4625523a9..7a362c727 100644 --- a/dascore/io/prodml/utils.py +++ b/dascore/io/prodml/utils.py @@ -15,11 +15,11 @@ from dascore.core.coords import get_coord from dascore.exceptions import InvalidSpoolError, PatchError from dascore.io.utils import convert_attr_units, get_exact_coord +from dascore.models import UTF8Str from dascore.units import get_quantity_str from dascore.utils.hdf5 import encode_h5_strings from dascore.utils.io import _normalize_source_patch_ids from dascore.utils.misc import iterate, maybe_get_items, register_func, unbyte -from dascore.utils.models import UTF8Str # --- Getting format/version diff --git a/dascore/io/sintela/protobuf_utils.py b/dascore/io/sintela/protobuf_utils.py index fdcaf67ce..6b6cd8828 100644 --- a/dascore/io/sintela/protobuf_utils.py +++ b/dascore/io/sintela/protobuf_utils.py @@ -56,8 +56,8 @@ from dascore.core.coords import get_coord from dascore.exceptions import InvalidFiberFileError from dascore.io.core import ScanPayload, make_scan_payload +from dascore.models import DascoreBaseModel, PositiveFiniteFloat, PositiveInt from dascore.utils.misc import optional_import, suppress_warnings -from dascore.utils.models import DascoreBaseModel, PositiveFiniteFloat, PositiveInt PBUF_MAGIC = 0x46554250 META_TAG = "META" diff --git a/dascore/io/utils.py b/dascore/io/utils.py index 7ef41de61..b647633e6 100644 --- a/dascore/io/utils.py +++ b/dascore/io/utils.py @@ -13,9 +13,9 @@ from dascore.core.coordmanager import CoordManager from dascore.core.coords import BaseCoord, CoordSegmented, get_coord from dascore.exceptions import CoordError, UnitError +from dascore.models import ArrayLike from dascore.units import convert_units, get_quantity_str from dascore.utils.misc import unbyte -from dascore.utils.models import ArrayLike # Stored coordinate arrays often carry sub-step jitter (e.g. GPS-stamped DAS # time). ``CoordSegmented.from_array`` treats every isolated sampling change as diff --git a/dascore/io/xml_binary/core.py b/dascore/io/xml_binary/core.py index 81c7c00b8..8cd8aeb9a 100644 --- a/dascore/io/xml_binary/core.py +++ b/dascore/io/xml_binary/core.py @@ -10,7 +10,7 @@ import dascore as dc from dascore.io import FiberIO, ScanPayload -from dascore.utils.models import UTF8Str +from dascore.models import UTF8Str from dascore.utils.paths import coerce_to_upath from .utils import _load_patches, _paths_to_scan_patches, _read_xml_metadata diff --git a/dascore/io/xml_binary/utils.py b/dascore/io/xml_binary/utils.py index b56d1ea11..23fb8a21b 100644 --- a/dascore/io/xml_binary/utils.py +++ b/dascore/io/xml_binary/utils.py @@ -8,7 +8,7 @@ import numpy as np import pandas as pd -from pydantic import ConfigDict +from pydantic import BaseModel, ConfigDict from pydantic.alias_generators import to_pascal import dascore as dc @@ -16,8 +16,8 @@ from dascore.core import get_coord, get_coord_manager from dascore.io import ScanPayload from dascore.io.core import make_scan_payload +from dascore.models import DateTime64 from dascore.utils.misc import iterate -from dascore.utils.models import BaseModel, DateTime64 from dascore.utils.pd import adjust_segments, filter_df from dascore.utils.remote_io import ensure_local_file from dascore.utils.time import to_float diff --git a/dascore/models/__init__.py b/dascore/models/__init__.py new file mode 100644 index 000000000..af0fd5a92 --- /dev/null +++ b/dascore/models/__init__.py @@ -0,0 +1,53 @@ +""" +DASCore's model layer: base classes and the annotated types they are built from. + +Anything which inherits from [DascoreBaseModel](`dascore.models.base.DascoreBaseModel`) +can appear in a DASCore document. Models used only to shuttle values inside one +module are plain pydantic models instead. +""" + +from __future__ import annotations + +from dascore.models.base import ( + DascoreBaseModel, + InventoryModel, + TimeRangedModel, + sensible_model_equals, + sensible_model_hash, + values_equal, +) +from dascore.models.types import ( + ArrayLike, + CommaSeparatedStr, + DateTime64, + DTypeLike, + FrozenDictType, + PositiveFiniteFloat, + PositiveInt, + TimeDelta64, + UnitQuantity, + UTF8Str, + frozen_dict_serializer, + frozen_dict_validator, +) + +__all__ = [ + "ArrayLike", + "CommaSeparatedStr", + "DTypeLike", + "DascoreBaseModel", + "DateTime64", + "FrozenDictType", + "InventoryModel", + "PositiveFiniteFloat", + "PositiveInt", + "TimeDelta64", + "TimeRangedModel", + "UTF8Str", + "UnitQuantity", + "frozen_dict_serializer", + "frozen_dict_validator", + "sensible_model_equals", + "sensible_model_hash", + "values_equal", +] diff --git a/dascore/models/base.py b/dascore/models/base.py new file mode 100644 index 000000000..d516b35df --- /dev/null +++ b/dascore/models/base.py @@ -0,0 +1,221 @@ +"""Base classes for DASCore's pydantic models.""" + +from __future__ import annotations + +from collections.abc import Mapping +from functools import cached_property + +import numpy as np +import pandas as pd +from pydantic import ( + BaseModel, + ConfigDict, + Field, + model_validator, +) +from typing_extensions import Self + +from dascore.compat import is_array_like +from dascore.exceptions import InvalidInventoryError +from dascore.models.types import DateTime64, FrozenDictType +from dascore.utils.misc import _all_null, all_close +from dascore.utils.time import to_datetime64 + + +def sensible_model_equals(self: BaseModel | Mapping, other: object) -> bool: + """Custom equality to not compare private attrs and handle numpy arrays.""" + d1 = self.model_dump() if isinstance(self, BaseModel) else self + if isinstance(other, BaseModel): + d2 = other.model_dump() + elif isinstance(other, Mapping): + d2 = other + else: # nothing else can carry the same fields + return NotImplemented + if not set(d1) == set(d2): # different keys, not equal + return False + for name in set(x for x in d1 if not x.startswith("_")): + # skip any private attributes. + if not values_equal(d1[name], d2[name]): + return False + return True + + +def values_equal(val1, val2) -> bool: + """Recursively compare dumped values; nulls are equal only to nulls.""" + if is_array_like(val1) or is_array_like(val2): + arr1, arr2 = np.asarray(val1), np.asarray(val2) + if arr1.shape != arr2.shape: + return False + if not np.array_equal(pd.isnull(arr1), pd.isnull(arr2)): + return False + return bool(all_close(arr1, arr2)) + if isinstance(val1, Mapping) and isinstance(val2, Mapping): + if set(val1) != set(val2): + return False + return all(values_equal(val1[key], val2[key]) for key in val1) + if isinstance(val1, list | tuple) and isinstance(val2, list | tuple): + if len(val1) != len(val2): + return False + return all(values_equal(v1, v2) for v1, v2 in zip(val1, val2)) + return bool(val1 == val2 or (_all_null(val1) and _all_null(val2))) + + +def _hash_key(value): + """Map a value onto one that hashes the way values_equal compares.""" + if value is None or isinstance(value, str | int): + return value + # Nulls count as equal to one another above, but every nan and NaT is a + # fresh object and both hash by identity, so they collapse to one key. + if isinstance(value, float): + return None if value != value else value + if isinstance(value, np.datetime64 | np.timedelta64): + return None if np.isnat(value) else value + # Mappings are compared without regard to order. + if isinstance(value, Mapping): + return frozenset((k, _hash_key(v)) for k, v in value.items()) + if isinstance(value, tuple): + return tuple(_hash_key(v) for v in value) + return value + + +def sensible_model_hash(self: BaseModel) -> int: + """Hash a model on its fields, agreeing with sensible_model_equals.""" + # Keyed by name and unordered, because equality compares the field names + # it finds rather than the order they were declared in. + fields = type(self).model_fields + return hash(frozenset((x, _hash_key(getattr(self, x))) for x in fields)) + + +class DascoreBaseModel(BaseModel): + """A base model with sensible configurations.""" + + _cache = {} + + model_config = ConfigDict( + extra="ignore", # TODO: change to raise, then let subclass overwrite + validate_assignment=True, + ignored_types=(cached_property,), + frozen=True, + validate_default=True, + arbitrary_types_allowed=True, + ) + + def new(self, **kwargs) -> Self: + """Create new instance with some attributed updated.""" + out = self.model_dump(exclude_unset=True) + out.update(kwargs) + return self.__class__(**out) + + @classmethod + def get_summary_df(cls): + """Get dataframe of attributes and descriptions for display.""" + fields = cls.model_fields + names_desc = { + i: v.description + for i, v in fields.items() + if getattr(v, "description", False) + } + out = pd.Series(names_desc).to_frame(name="description") + out.index.name = "attribute" + return out + + __eq__ = sensible_model_equals + # Defined together: pydantic would otherwise derive a hash straight from + # the field values, which disagrees with how __eq__ treats nulls. + __hash__ = sensible_model_hash + + +class InventoryModel(DascoreBaseModel): + """ + Base class for immutable DASDAE inventory objects. + + Every inventory object carries two uniform attachment points for + information the model does not otherwise represent: ``description`` + (free prose for humans, matching StationXML's Description element) + and ``extra_fields`` (typed key-values, e.g. for round-tripping + unmodeled metadata from external formats). + + Every field is immutable: collections are tuples and mappings are + frozen, so instances are safe to hold by reference. They hash on their + field values whenever those values are themselves hashable. + """ + + description: str = Field(default="", description="Free-text description.") + extra_fields: FrozenDictType[str, str | int | float | bool] = Field( + default_factory=dict, + description="Extra metadata not represented by standardized fields.", + ) + + model_config = ConfigDict( + extra="forbid", + frozen=True, + validate_assignment=True, + validate_default=True, + arbitrary_types_allowed=True, + ) + + def new(self, **kwargs) -> Self: + """ + Create a new instance with some attributes updated. + + Dumps all fields (not just the set ones) so union discriminators and + validator-normalized fields survive reconstruction. + """ + out = self.model_dump() + out.update(kwargs) + return self.__class__(**out) + + +class TimeRangedModel(InventoryModel): + """Base class for inventory objects with time-validity epochs. + + Validity intervals are half-open, ``[start_time, end_time)``; an unset + (NaT) end time means the epoch is ongoing. All times are UTC. + """ + + start_time: DateTime64 = Field( + default=np.datetime64("NaT", "ns"), + description="Start time for which this metadata item is valid (UTC).", + ) + end_time: DateTime64 = Field( + default=np.datetime64("NaT", "ns"), + description=( + "End time for which this metadata item is valid (UTC); NaT while ongoing." + ), + ) + + @model_validator(mode="after") + def _check_time_order(self): + """A set end time must follow the start time.""" + start, end = self.start_time, self.end_time + if not pd.isnull(start) and not pd.isnull(end) and end <= start: + msg = f"end_time {end} must be after start_time {start}." + raise InvalidInventoryError(msg) + return self + + def is_effective_at(self, time) -> bool: + """Return True if this epoch is valid at the supplied time (half-open).""" + time = to_datetime64(time) + if pd.isnull(time): + return True + start = self.start_time + end = self.end_time + after_start = pd.isnull(start) or start <= time + before_end = pd.isnull(end) or time < end + return bool(after_start and before_end) + + def overlaps(self, other: TimeRangedModel) -> bool: + """ + Return True if two half-open validity intervals overlap. + + Unset (NaT) starts are unbounded past; unset ends are ongoing. + """ + s1, e1, s2, e2 = ( + self.start_time, + self.end_time, + other.start_time, + other.end_time, + ) + first_starts_before = pd.isnull(e2) or pd.isnull(s1) or s1 < e2 + second_starts_before = pd.isnull(e1) or pd.isnull(s2) or s2 < e1 + return bool(first_starts_before and second_starts_before) diff --git a/dascore/models/types.py b/dascore/models/types.py new file mode 100644 index 000000000..abbbb9ae8 --- /dev/null +++ b/dascore/models/types.py @@ -0,0 +1,99 @@ +"""Annotated types with DASCore's validation and serialization attached.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Annotated, TypeVar + +import numpy as np +from pydantic import ( + AfterValidator, + Field, + PlainSerializer, + PlainValidator, +) + +from dascore.compat import array +from dascore.units import Quantity, get_quantity, get_quantity_str +from dascore.utils.mapping import FrozenDict +from dascore.utils.misc import to_str, unbyte +from dascore.utils.time import to_datetime64, to_timedelta64 + +# --- A list of custom types with appropriate serialization/deserialization +# these can just be use with pydantic type-hints. + +# Freezes without validating contents. Use FrozenDictType below when the +# declared value types must still be enforced. +frozen_dict_validator = PlainValidator(lambda x: FrozenDict(x)) +frozen_dict_serializer = PlainSerializer(lambda x: dict(x)) + +# A datetime64 +DateTime64 = Annotated[ + np.datetime64, + PlainValidator(to_datetime64), + PlainSerializer(to_str, when_used="json"), # getting undefined name +] + +TimeDelta64 = Annotated[ + np.timedelta64, + PlainValidator(to_timedelta64), + PlainSerializer(to_str, when_used="json"), # getting undefined name +] + +# The validator may preserve non-numpy array-likes (see compat.array), but +# ndarray is deliberately the single static face of array values; a structural +# protocol is not worth the complexity it spreads through every signature. +ArrayLike = Annotated[ + np.ndarray, + PlainValidator(array), +] + +DTypeLike = Annotated[ + str, + PlainValidator(np.dtype), +] + + +def _to_unit_quantity(value): + """Read units, refusing a quantity that carries many magnitudes.""" + out = get_quantity(value) + try: + # Passing a sequence makes pint build an array magnitude, which is + # writable through the frozen model holding it. Asking whether it can + # be hashed is the direct question; no real unit spelling fails it. + hash(out) + except TypeError: + msg = f"Units must name a single unit, got {value!r}." + raise ValueError(msg) from None + return out + + +UnitQuantity = Annotated[ + Quantity | str | None, + PlainValidator(_to_unit_quantity), + PlainSerializer(get_quantity_str), +] + +CommaSeparatedStr = Annotated[ + str, PlainValidator(lambda x: x if isinstance(x, str) else ",".join(x)) +] + +K = TypeVar("K") +V = TypeVar("V") + +# Mapping, not dict: the runtime value is a FrozenDict, so a dict annotation +# would let type checkers pass writes that raise. AfterValidator, not +# PlainValidator: a plain validator would replace the declared value-type check. +FrozenDictType = Annotated[ + Mapping[K, V], + AfterValidator(lambda x: FrozenDict(x)), + PlainSerializer(dict), +] + +UTF8Str = Annotated[str, PlainValidator(unbyte)] + +# A positive (> 0) integer. +PositiveInt = Annotated[int, Field(gt=0)] + +# A positive (> 0), finite (no nan/inf) float. +PositiveFiniteFloat = Annotated[float, Field(gt=0, allow_inf_nan=False)] diff --git a/dascore/proc/basic.py b/dascore/proc/basic.py index 5254a4996..9aed6236f 100644 --- a/dascore/proc/basic.py +++ b/dascore/proc/basic.py @@ -19,9 +19,9 @@ ) from dascore.core.coords import get_coord from dascore.exceptions import ParameterError +from dascore.models import ArrayLike from dascore.utils.array import _apply_binary_ufunc from dascore.utils.misc import _get_nullish -from dascore.utils.models import ArrayLike from dascore.utils.patch import ( align_patch_coords, get_dim_axis_value, diff --git a/dascore/proc/inventory.py b/dascore/proc/inventory.py index 7c26654b9..dee32c061 100644 --- a/dascore/proc/inventory.py +++ b/dascore/proc/inventory.py @@ -35,12 +35,12 @@ PatchError, UnresolvedPatchError, ) +from dascore.models import values_equal from dascore.proc.coords import update_coords from dascore.units import get_quantity_str from dascore.utils.attrs import validate_conflict from dascore.utils.docs import compose_docstring from dascore.utils.misc import iterate, validate_acquisition_key -from dascore.utils.models import values_equal from dascore.utils.patch import patch_function from dascore.utils.time import to_datetime64 diff --git a/dascore/proc/mute.py b/dascore/proc/mute.py index 04bbbd221..78e3046e6 100644 --- a/dascore/proc/mute.py +++ b/dascore/proc/mute.py @@ -14,11 +14,11 @@ import dascore as dc from dascore.constants import PatchType from dascore.exceptions import ParameterError +from dascore.models import DascoreBaseModel from dascore.utils.docs import compose_docstring from dascore.utils.misc import ( get_2d_line_intersection, ) -from dascore.utils.models import DascoreBaseModel from dascore.utils.patch import get_dim_axis_value, patch_function _smooth_param = """ diff --git a/dascore/utils/array.py b/dascore/utils/array.py index 51bd20185..19412c630 100644 --- a/dascore/utils/array.py +++ b/dascore/utils/array.py @@ -15,9 +15,9 @@ from dascore.compat import array, is_array from dascore.constants import DEFAULT_ATTRS_TO_IGNORE, PatchType from dascore.exceptions import ParameterError, PatchBroadcastError, UnitError +from dascore.models import ArrayLike from dascore.units import DimensionalityError, Quantity, Unit, get_quantity from dascore.utils.misc import iterate -from dascore.utils.models import ArrayLike from dascore.utils.patch import ( _merge_aligned_coords, _merge_models, diff --git a/dascore/utils/coordmanager.py b/dascore/utils/coordmanager.py index 504a7e520..e176aebe3 100644 --- a/dascore/utils/coordmanager.py +++ b/dascore/utils/coordmanager.py @@ -10,8 +10,8 @@ import dascore as dc from dascore.exceptions import CoordMergeError +from dascore.models import ArrayLike from dascore.utils.display import get_nice_text -from dascore.utils.models import ArrayLike def merge_coord_managers( diff --git a/dascore/utils/models.py b/dascore/utils/models.py index 64d67c95d..2c0186bec 100644 --- a/dascore/utils/models.py +++ b/dascore/utils/models.py @@ -1,305 +1,55 @@ -"""Utilities for models.""" +""" +Deprecated home of DASCore's models; import from [dascore.models](`dascore.models`). + +Everything here is re-exported from its new home so out-of-tree readers which +import from this path keep working. +""" from __future__ import annotations -from collections.abc import Mapping -from functools import cached_property -from typing import Annotated, TypeVar +from pydantic import BaseModel -import numpy as np -import pandas as pd -from pydantic import ( - AfterValidator, - BaseModel, - ConfigDict, - Field, - PlainSerializer, - PlainValidator, - model_validator, +from dascore.models.base import ( + DascoreBaseModel, + InventoryModel, + TimeRangedModel, + sensible_model_equals, + sensible_model_hash, + values_equal, +) +from dascore.models.types import ( + ArrayLike, + CommaSeparatedStr, + DateTime64, + DTypeLike, + FrozenDictType, + PositiveFiniteFloat, + PositiveInt, + TimeDelta64, + UnitQuantity, + UTF8Str, + frozen_dict_serializer, + frozen_dict_validator, ) -from typing_extensions import Self - -from dascore.compat import array, is_array_like -from dascore.exceptions import InvalidInventoryError -from dascore.units import Quantity, get_quantity, get_quantity_str -from dascore.utils.mapping import FrozenDict -from dascore.utils.misc import _all_null, all_close, to_str, unbyte -from dascore.utils.time import to_datetime64, to_timedelta64 - -# --- A list of custom types with appropriate serialization/deserialization -# these can just be use with pydantic type-hints. - -# Freezes without validating contents. Use FrozenDictType below when the -# declared value types must still be enforced. -frozen_dict_validator = PlainValidator(lambda x: FrozenDict(x)) -frozen_dict_serializer = PlainSerializer(lambda x: dict(x)) - -# A datetime64 -DateTime64 = Annotated[ - np.datetime64, - PlainValidator(to_datetime64), - PlainSerializer(to_str, when_used="json"), # getting undefined name -] - -TimeDelta64 = Annotated[ - np.timedelta64, - PlainValidator(to_timedelta64), - PlainSerializer(to_str, when_used="json"), # getting undefined name -] - -# The validator may preserve non-numpy array-likes (see compat.array), but -# ndarray is deliberately the single static face of array values; a structural -# protocol is not worth the complexity it spreads through every signature. -ArrayLike = Annotated[ - np.ndarray, - PlainValidator(array), -] - -DTypeLike = Annotated[ - str, - PlainValidator(np.dtype), -] - - -def _to_unit_quantity(value): - """Read units, refusing a quantity that carries many magnitudes.""" - out = get_quantity(value) - try: - # Passing a sequence makes pint build an array magnitude, which is - # writable through the frozen model holding it. Asking whether it can - # be hashed is the direct question; no real unit spelling fails it. - hash(out) - except TypeError: - msg = f"Units must name a single unit, got {value!r}." - raise ValueError(msg) from None - return out - - -UnitQuantity = Annotated[ - Quantity | str | None, - PlainValidator(_to_unit_quantity), - PlainSerializer(get_quantity_str), -] - -CommaSeparatedStr = Annotated[ - str, PlainValidator(lambda x: x if isinstance(x, str) else ",".join(x)) -] - -K = TypeVar("K") -V = TypeVar("V") -# Mapping, not dict: the runtime value is a FrozenDict, so a dict annotation -# would let type checkers pass writes that raise. AfterValidator, not -# PlainValidator: a plain validator would replace the declared value-type check. -FrozenDictType = Annotated[ - Mapping[K, V], - AfterValidator(lambda x: FrozenDict(x)), - PlainSerializer(dict), +__all__ = [ + "ArrayLike", + "BaseModel", + "CommaSeparatedStr", + "DTypeLike", + "DascoreBaseModel", + "DateTime64", + "FrozenDictType", + "InventoryModel", + "PositiveFiniteFloat", + "PositiveInt", + "TimeDelta64", + "TimeRangedModel", + "UTF8Str", + "UnitQuantity", + "frozen_dict_serializer", + "frozen_dict_validator", + "sensible_model_equals", + "sensible_model_hash", + "values_equal", ] - -UTF8Str = Annotated[str, PlainValidator(unbyte)] - -# A positive (> 0) integer. -PositiveInt = Annotated[int, Field(gt=0)] - -# A positive (> 0), finite (no nan/inf) float. -PositiveFiniteFloat = Annotated[float, Field(gt=0, allow_inf_nan=False)] - - -def sensible_model_equals(self: BaseModel | Mapping, other: object) -> bool: - """Custom equality to not compare private attrs and handle numpy arrays.""" - d1 = self.model_dump() if isinstance(self, BaseModel) else self - if isinstance(other, BaseModel): - d2 = other.model_dump() - elif isinstance(other, Mapping): - d2 = other - else: # nothing else can carry the same fields - return NotImplemented - if not set(d1) == set(d2): # different keys, not equal - return False - for name in set(x for x in d1 if not x.startswith("_")): - # skip any private attributes. - if not values_equal(d1[name], d2[name]): - return False - return True - - -def values_equal(val1, val2) -> bool: - """Recursively compare dumped values; nulls are equal only to nulls.""" - if is_array_like(val1) or is_array_like(val2): - arr1, arr2 = np.asarray(val1), np.asarray(val2) - if arr1.shape != arr2.shape: - return False - if not np.array_equal(pd.isnull(arr1), pd.isnull(arr2)): - return False - return bool(all_close(arr1, arr2)) - if isinstance(val1, Mapping) and isinstance(val2, Mapping): - if set(val1) != set(val2): - return False - return all(values_equal(val1[key], val2[key]) for key in val1) - if isinstance(val1, list | tuple) and isinstance(val2, list | tuple): - if len(val1) != len(val2): - return False - return all(values_equal(v1, v2) for v1, v2 in zip(val1, val2)) - return bool(val1 == val2 or (_all_null(val1) and _all_null(val2))) - - -def _hash_key(value): - """Map a value onto one that hashes the way values_equal compares.""" - if value is None or isinstance(value, str | int): - return value - # Nulls count as equal to one another above, but every nan and NaT is a - # fresh object and both hash by identity, so they collapse to one key. - if isinstance(value, float): - return None if value != value else value - if isinstance(value, np.datetime64 | np.timedelta64): - return None if np.isnat(value) else value - # Mappings are compared without regard to order. - if isinstance(value, Mapping): - return frozenset((k, _hash_key(v)) for k, v in value.items()) - if isinstance(value, tuple): - return tuple(_hash_key(v) for v in value) - return value - - -def sensible_model_hash(self: BaseModel) -> int: - """Hash a model on its fields, agreeing with sensible_model_equals.""" - # Keyed by name and unordered, because equality compares the field names - # it finds rather than the order they were declared in. - fields = type(self).model_fields - return hash(frozenset((x, _hash_key(getattr(self, x))) for x in fields)) - - -class DascoreBaseModel(BaseModel): - """A base model with sensible configurations.""" - - _cache = {} - - model_config = ConfigDict( - extra="ignore", # TODO: change to raise, then let subclass overwrite - validate_assignment=True, - ignored_types=(cached_property,), - frozen=True, - validate_default=True, - arbitrary_types_allowed=True, - ) - - def new(self, **kwargs) -> Self: - """Create new instance with some attributed updated.""" - out = self.model_dump(exclude_unset=True) - out.update(kwargs) - return self.__class__(**out) - - @classmethod - def get_summary_df(cls): - """Get dataframe of attributes and descriptions for display.""" - fields = cls.model_fields - names_desc = { - i: v.description - for i, v in fields.items() - if getattr(v, "description", False) - } - out = pd.Series(names_desc).to_frame(name="description") - out.index.name = "attribute" - return out - - __eq__ = sensible_model_equals - # Defined together: pydantic would otherwise derive a hash straight from - # the field values, which disagrees with how __eq__ treats nulls. - __hash__ = sensible_model_hash - - -class InventoryModel(DascoreBaseModel): - """ - Base class for immutable DASDAE inventory objects. - - Every inventory object carries two uniform attachment points for - information the model does not otherwise represent: ``description`` - (free prose for humans, matching StationXML's Description element) - and ``extra_fields`` (typed key-values, e.g. for round-tripping - unmodeled metadata from external formats). - - Every field is immutable: collections are tuples and mappings are - frozen, so instances are safe to hold by reference. They hash on their - field values whenever those values are themselves hashable. - """ - - description: str = Field(default="", description="Free-text description.") - extra_fields: FrozenDictType[str, str | int | float | bool] = Field( - default_factory=dict, - description="Extra metadata not represented by standardized fields.", - ) - - model_config = ConfigDict( - extra="forbid", - frozen=True, - validate_assignment=True, - validate_default=True, - arbitrary_types_allowed=True, - ) - - def new(self, **kwargs) -> Self: - """ - Create a new instance with some attributes updated. - - Dumps all fields (not just the set ones) so union discriminators and - validator-normalized fields survive reconstruction. - """ - out = self.model_dump() - out.update(kwargs) - return self.__class__(**out) - - -class TimeRangedModel(InventoryModel): - """Base class for inventory objects with time-validity epochs. - - Validity intervals are half-open, ``[start_time, end_time)``; an unset - (NaT) end time means the epoch is ongoing. All times are UTC. - """ - - start_time: DateTime64 = Field( - default=np.datetime64("NaT", "ns"), - description="Start time for which this metadata item is valid (UTC).", - ) - end_time: DateTime64 = Field( - default=np.datetime64("NaT", "ns"), - description=( - "End time for which this metadata item is valid (UTC); NaT while ongoing." - ), - ) - - @model_validator(mode="after") - def _check_time_order(self): - """A set end time must follow the start time.""" - start, end = self.start_time, self.end_time - if not pd.isnull(start) and not pd.isnull(end) and end <= start: - msg = f"end_time {end} must be after start_time {start}." - raise InvalidInventoryError(msg) - return self - - def is_effective_at(self, time) -> bool: - """Return True if this epoch is valid at the supplied time (half-open).""" - time = to_datetime64(time) - if pd.isnull(time): - return True - start = self.start_time - end = self.end_time - after_start = pd.isnull(start) or start <= time - before_end = pd.isnull(end) or time < end - return bool(after_start and before_end) - - def overlaps(self, other: TimeRangedModel) -> bool: - """ - Return True if two half-open validity intervals overlap. - - Unset (NaT) starts are unbounded past; unset ends are ongoing. - """ - s1, e1, s2, e2 = ( - self.start_time, - self.end_time, - other.start_time, - other.end_time, - ) - first_starts_before = pd.isnull(e2) or pd.isnull(s1) or s1 < e2 - second_starts_before = pd.isnull(e1) or pd.isnull(s2) or s2 < e1 - return bool(first_starts_before and second_starts_before) diff --git a/tests/test_core/test_inventory.py b/tests/test_core/test_inventory.py index 77caba3d3..fd24754b2 100644 --- a/tests/test_core/test_inventory.py +++ b/tests/test_core/test_inventory.py @@ -14,8 +14,8 @@ from dascore.core import Inventory from dascore.core import inventory as inv from dascore.exceptions import InvalidInventoryError +from dascore.models import values_equal from dascore.utils.mapping import FrozenDict -from dascore.utils.models import values_equal def build_inventory() -> inv.Inventory: diff --git a/tests/test_core/test_inventory_loader.py b/tests/test_core/test_inventory_loader.py index 8bf316f1e..4a5c530b8 100644 --- a/tests/test_core/test_inventory_loader.py +++ b/tests/test_core/test_inventory_loader.py @@ -16,7 +16,7 @@ InvalidInventoryError, MissingOptionalDependencyError, ) -from dascore.utils.models import InventoryModel, TimeRangedModel +from dascore.models import InventoryModel, TimeRangedModel pytest.importorskip("yaml") diff --git a/tests/test_utils/test_models.py b/tests/test_models.py similarity index 99% rename from tests/test_utils/test_models.py rename to tests/test_models.py index d7a13da0c..a5a149f73 100644 --- a/tests/test_utils/test_models.py +++ b/tests/test_models.py @@ -8,8 +8,7 @@ import pytest from pydantic import Field, ValidationError -from dascore.units import Quantity -from dascore.utils.models import ( +from dascore.models import ( DascoreBaseModel, DateTime64, FrozenDictType, @@ -17,6 +16,7 @@ UnitQuantity, sensible_model_equals, ) +from dascore.units import Quantity class _TestModel(DascoreBaseModel): From 303f844defa4e40c5600825eb6922ca5fcf64649 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Thu, 13 Aug 2026 12:18:48 +0200 Subject: [PATCH 2/8] Take the parsing-only models off DascoreBaseModel The mute geometries and the Sintela protobuf parsers want validation and nothing else the base offers: they carry values between two functions inside one module and are never serialized. Leaving them on the base would enroll them in a document machinery none of them participate in. --- dascore/io/sintela/protobuf_utils.py | 24 ++++++++++++++++++------ dascore/proc/mute.py | 10 ++++++++-- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/dascore/io/sintela/protobuf_utils.py b/dascore/io/sintela/protobuf_utils.py index 6b6cd8828..6d5832286 100644 --- a/dascore/io/sintela/protobuf_utils.py +++ b/dascore/io/sintela/protobuf_utils.py @@ -48,7 +48,7 @@ from typing import Any import numpy as np -from pydantic import ValidationError +from pydantic import BaseModel, ConfigDict, ValidationError import dascore as dc from dascore.core.attrs import PatchAttrs @@ -56,7 +56,7 @@ from dascore.core.coords import get_coord from dascore.exceptions import InvalidFiberFileError from dascore.io.core import ScanPayload, make_scan_payload -from dascore.models import DascoreBaseModel, PositiveFiniteFloat, PositiveInt +from dascore.models import PositiveFiniteFloat, PositiveInt from dascore.utils.misc import optional_import, suppress_warnings PBUF_MAGIC = 0x46554250 @@ -153,14 +153,26 @@ class SintelaProtobufAttrs(PatchAttrs): demod_data_type: str = "" -class EnvelopeRecord(DascoreBaseModel): +class _ProtobufModel(BaseModel): + """ + Base for this module's parsing models. + + Plain pydantic: these validate values on the way out of a protobuf + payload and are never serialized, so they want none of what + DascoreBaseModel adds beyond validation. + """ + + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + +class EnvelopeRecord(_ProtobufModel): """The envelope information for one MTLV record.""" tag: str payload: bytes -class ParsedMeta(DascoreBaseModel): +class ParsedMeta(_ProtobufModel): """Selected metadata fields promoted from META packets.""" recorder_namespace: str = "" @@ -823,7 +835,7 @@ def _decode_family(parsed: list[tuple[str, Any]], meta: ParsedMeta): return family_cls.from_parsed(parsed, meta).decode(parsed) -class _PacketHeaderFields(DascoreBaseModel): +class _PacketHeaderFields(_ProtobufModel): """ Validated per-packet header fields shared by all families. @@ -871,7 +883,7 @@ def _reduce_common(fields: list[_PacketHeaderFields]): ) -class _PacketMetadata(DascoreBaseModel): +class _PacketMetadata(_ProtobufModel): """ Base for a validated, decodable Sintela packet family. diff --git a/dascore/proc/mute.py b/dascore/proc/mute.py index 78e3046e6..5b9d2a17c 100644 --- a/dascore/proc/mute.py +++ b/dascore/proc/mute.py @@ -9,12 +9,12 @@ import numpy as np from numpy.linalg import norm from numpy.typing import NDArray +from pydantic import BaseModel, ConfigDict from scipy.ndimage import gaussian_filter import dascore as dc from dascore.constants import PatchType from dascore.exceptions import ParameterError -from dascore.models import DascoreBaseModel from dascore.utils.docs import compose_docstring from dascore.utils.misc import ( get_2d_line_intersection, @@ -40,11 +40,17 @@ _smooth_type = None | float | int | tuple[float | int, ...] | dict[str, float | int] -class _MuteGeometry(ABC, DascoreBaseModel): +class _MuteGeometry(ABC, BaseModel): """ Parent class for Mute Geometry. + + A plain pydantic model: these carry a mute's geometry from the argument + parsing to the mask and are never serialized, so they want validation + and nothing else DascoreBaseModel offers. """ + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + dims: tuple[str, ...] axes: tuple[int, ...] relative: bool = True From a8ae188b6f3140607e84f75de9eeb919dde7b590 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Thu, 13 Aug 2026 12:22:29 +0200 Subject: [PATCH 3/8] Let every PatchAttrs subclass read back what it writes A float defaulting to nan writes null and then refuses to read it, so twelve format classes could not reconstruct from their own json. An optional number is spelled FiniteFloat | None, which the inventory models already used and which nan cannot enter; FiniteFloat moves beside the other shared types. The walk covers formats added later. --- dascore/core/inventory.py | 3 +- dascore/io/ai4eps/core.py | 12 ++--- dascore/io/ap_sensing/core.py | 7 ++- dascore/io/febus/core.py | 8 ++-- dascore/io/odh4/core.py | 7 ++- dascore/io/optodas/core.py | 6 +-- dascore/io/prodml/utils.py | 10 ++--- dascore/io/silixah5/core.py | 7 ++- dascore/io/sintela/core.py | 3 +- dascore/io/sintela/protobuf_utils.py | 4 +- dascore/io/sr4731/utils.py | 9 ++-- dascore/io/xml_binary/core.py | 7 ++- dascore/models/__init__.py | 2 + dascore/models/types.py | 5 +++ dascore/utils/models.py | 2 + tests/test_models.py | 66 ++++++++++++++++++++++++++++ 16 files changed, 113 insertions(+), 45 deletions(-) diff --git a/dascore/core/inventory.py b/dascore/core/inventory.py index 907798b4b..80b4bafbb 100644 --- a/dascore/core/inventory.py +++ b/dascore/core/inventory.py @@ -46,6 +46,7 @@ from dascore.exceptions import InvalidInventoryError, ParameterError from dascore.models import ( DateTime64, + FiniteFloat, FrozenDictType, InventoryModel, TimeRangedModel, @@ -90,8 +91,6 @@ # The token rule is shared with PatchAttrs.acquisition_key so a code legal # in one is legal in the other. CodeStr = Annotated[str, AfterValidator(check_code)] -# A float which must be finite; nan/inf silently poison downstream math. -FiniteFloat = Annotated[float, Field(allow_inf_nan=False)] # Sensor orientation, in the ranges seismology already uses. Azimuth = Annotated[float, Field(ge=0, lt=360, allow_inf_nan=False)] Dip = Annotated[float, Field(ge=-90, le=90, allow_inf_nan=False)] diff --git a/dascore/io/ai4eps/core.py b/dascore/io/ai4eps/core.py index 364fb6561..659e3efb3 100644 --- a/dascore/io/ai4eps/core.py +++ b/dascore/io/ai4eps/core.py @@ -9,7 +9,7 @@ import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload -from dascore.models import DateTime64 +from dascore.models import DateTime64, FiniteFloat from dascore.utils.hdf5 import H5Reader from .utils import _get_attrs_dict, _get_coords, _get_patches, _is_ai4eps @@ -19,12 +19,12 @@ class AI4EPSPatchAttrs(dc.PatchAttrs): """Patch attributes for AI4EPS event files.""" event_id: str = "" - event_time: DateTime64 = np.datetime64("NaT") - magnitude: float = np.nan + event_time: DateTime64 = np.datetime64("NaT", "ns") + magnitude: FiniteFloat | None = None magnitude_type: str = "" - event_latitude: float = np.nan - event_longitude: float = np.nan - event_depth_km: float = np.nan + event_latitude: FiniteFloat | None = None + event_longitude: FiniteFloat | None = None + event_depth_km: FiniteFloat | None = None class AI4EPSV1(FiberIO): diff --git a/dascore/io/ap_sensing/core.py b/dascore/io/ap_sensing/core.py index 09498c239..d2d176a10 100644 --- a/dascore/io/ap_sensing/core.py +++ b/dascore/io/ap_sensing/core.py @@ -6,11 +6,10 @@ from typing import Literal -import numpy as np - import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload +from dascore.models import FiniteFloat from dascore.utils.hdf5 import H5Reader from .utils import _get_attrs_dict, _get_coords, _get_patches, _get_version_string @@ -19,8 +18,8 @@ class APSensingPatchAttrs(dc.PatchAttrs): """Patch Attributes for AP sensing.""" - gauge_length: float = np.nan - radians_to_nano_strain: float = np.nan + gauge_length: FiniteFloat | None = None + radians_to_nano_strain: FiniteFloat | None = None class APSensingV10(FiberIO): diff --git a/dascore/io/febus/core.py b/dascore/io/febus/core.py index a9fe973f1..08319dcc7 100644 --- a/dascore/io/febus/core.py +++ b/dascore/io/febus/core.py @@ -7,8 +7,6 @@ import warnings from typing import Literal -import numpy as np - import dascore as dc from dascore.constants import ( float_select_type, @@ -18,7 +16,7 @@ ) from dascore.io import FiberIO, ScanPayload from dascore.io.core import make_scan_payload -from dascore.models import UTF8Str +from dascore.models import FiniteFloat, UTF8Str from dascore.utils.hdf5 import H5Reader from dascore.utils.io import TextReader @@ -61,8 +59,8 @@ class FebusPatchAttrs(dc.PatchAttrs): The zone designations """ - gauge_length: float = np.nan - pulse_length: float = np.nan + gauge_length: FiniteFloat | None = None + pulse_length: FiniteFloat | None = None group: str = "" source: str = "" diff --git a/dascore/io/odh4/core.py b/dascore/io/odh4/core.py index b76befb5e..4b01eb841 100644 --- a/dascore/io/odh4/core.py +++ b/dascore/io/odh4/core.py @@ -4,11 +4,10 @@ from typing import Literal -import numpy as np - import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload +from dascore.models import FiniteFloat from dascore.utils.hdf5 import H5Reader from .utils import _get_attrs_dict, _get_coords, _get_patches, _is_odh4, _read_attrs @@ -17,8 +16,8 @@ class ODH4PatchAttrs(dc.PatchAttrs): """Patch attributes for ODH4 files.""" - gauge_length: float = np.nan - scale_factor_to_strain: float = np.nan + gauge_length: FiniteFloat | None = None + scale_factor_to_strain: FiniteFloat | None = None class ODH4V1(FiberIO): diff --git a/dascore/io/optodas/core.py b/dascore/io/optodas/core.py index 0797c2f9d..c756b3755 100644 --- a/dascore/io/optodas/core.py +++ b/dascore/io/optodas/core.py @@ -4,12 +4,10 @@ from typing import Literal -import numpy as np - import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload -from dascore.models import UTF8Str +from dascore.models import FiniteFloat, UTF8Str from dascore.utils.hdf5 import H5Reader from .utils import _get_opto_das_attrs, _get_opto_das_version_str, _read_opto_das @@ -18,7 +16,7 @@ class OptoDASPatchAttrs(dc.PatchAttrs): """Patch attrs for OptoDAS.""" - gauge_length: float = np.nan + gauge_length: FiniteFloat | None = None schema_version: UTF8Str = "" diff --git a/dascore/io/prodml/utils.py b/dascore/io/prodml/utils.py index 7a362c727..1251a5ff7 100644 --- a/dascore/io/prodml/utils.py +++ b/dascore/io/prodml/utils.py @@ -15,7 +15,7 @@ from dascore.core.coords import get_coord from dascore.exceptions import InvalidSpoolError, PatchError from dascore.io.utils import convert_attr_units, get_exact_coord -from dascore.models import UTF8Str +from dascore.models import FiniteFloat, UTF8Str from dascore.units import get_quantity_str from dascore.utils.hdf5 import encode_h5_strings from dascore.utils.io import _normalize_source_patch_ids @@ -77,8 +77,8 @@ class ProdMLRawPatchAttrs(dc.PatchAttrs): """Patch attrs for raw data contained in ProdML.""" - pulse_width: float = np.nan - gauge_length: float = np.nan + pulse_width: FiniteFloat | None = None + gauge_length: FiniteFloat | None = None schema_version: UTF8Str = "" @@ -86,13 +86,13 @@ class ProdMLFbePatchAttrs(ProdMLRawPatchAttrs): """Patch attrs for fbe (frequency band extracted) data in Prodml.""" raw_reference: UTF8Str = "" - transform_size: float = np.nan + transform_size: FiniteFloat | None = None transform_type: UTF8Str = "" window_size: int | None = None window_function: UTF8Str = "" window_overlap: int | None = None start_frequency: float = 0 - end_frequency: float = np.inf + end_frequency: FiniteFloat | None = None @dataclass diff --git a/dascore/io/silixah5/core.py b/dascore/io/silixah5/core.py index 8fefe4f87..c177b0d01 100644 --- a/dascore/io/silixah5/core.py +++ b/dascore/io/silixah5/core.py @@ -6,11 +6,10 @@ from typing import Literal -import numpy as np - import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload +from dascore.models import FiniteFloat from dascore.utils.hdf5 import H5Reader from .utils import ( @@ -26,8 +25,8 @@ class SilixaPatchAttrs(dc.PatchAttrs): """Patch Attributes for Silixa hdf5 format.""" - gauge_length: float = np.nan - pulse_width: float = np.nan + gauge_length: FiniteFloat | None = None + pulse_width: FiniteFloat | None = None class SilixaH5V1(FiberIO): diff --git a/dascore/io/sintela/core.py b/dascore/io/sintela/core.py index ca49844bc..e4245723c 100644 --- a/dascore/io/sintela/core.py +++ b/dascore/io/sintela/core.py @@ -11,6 +11,7 @@ import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload +from dascore.models import FiniteFloat from dascore.utils.io import BinaryReader, LocalBinaryReader from .protobuf_utils import get_supported_family_tag, read_payload, scan_payload @@ -26,7 +27,7 @@ class SintelaPatchAttrs(dc.PatchAttrs): """Patch Attributes for Sintela binary format.""" - gauge_length: float = np.nan + gauge_length: FiniteFloat | None = None class SintelaBinaryV3(FiberIO): diff --git a/dascore/io/sintela/protobuf_utils.py b/dascore/io/sintela/protobuf_utils.py index 6d5832286..34e574034 100644 --- a/dascore/io/sintela/protobuf_utils.py +++ b/dascore/io/sintela/protobuf_utils.py @@ -56,7 +56,7 @@ from dascore.core.coords import get_coord from dascore.exceptions import InvalidFiberFileError from dascore.io.core import ScanPayload, make_scan_payload -from dascore.models import PositiveFiniteFloat, PositiveInt +from dascore.models import FiniteFloat, PositiveFiniteFloat, PositiveInt from dascore.utils.misc import optional_import, suppress_warnings PBUF_MAGIC = 0x46554250 @@ -143,7 +143,7 @@ def _get_fft_data_type(has_complex: bool) -> dict[str, str]: class SintelaProtobufAttrs(PatchAttrs): """Patch attributes for Sintela protobuf recordings.""" - gauge_length: float = np.nan + gauge_length: FiniteFloat | None = None packet_type: str = "" recorder_namespace: str = "" metadata_recording_time: np.datetime64 | None = None diff --git a/dascore/io/sr4731/utils.py b/dascore/io/sr4731/utils.py index 85e6c38e9..ca1ac60fc 100644 --- a/dascore/io/sr4731/utils.py +++ b/dascore/io/sr4731/utils.py @@ -35,6 +35,7 @@ from dascore.exceptions import InvalidFiberFileError from dascore.io.core import ScanPayload, make_scan_payload from dascore.io.utils import build_patches +from dascore.models import FiniteFloat DIMS = ("time", "distance") REQUIRED_BLOCKS = frozenset( @@ -56,10 +57,10 @@ class Block: class SR4731PatchAttrs(PatchAttrs): """Patch attributes for supported SR-4731 SOR files.""" - wavelength_nm: float = np.nan - acquisition_range_m: float = np.nan - sample_spacing_usec: float = np.nan - refractive_index: float = np.nan + wavelength_nm: FiniteFloat | None = None + acquisition_range_m: FiniteFloat | None = None + sample_spacing_usec: FiniteFloat | None = None + refractive_index: FiniteFloat | None = None trace_count: int = 0 sample_scale: int = 0 diff --git a/dascore/io/xml_binary/core.py b/dascore/io/xml_binary/core.py index 8cd8aeb9a..f64b7f856 100644 --- a/dascore/io/xml_binary/core.py +++ b/dascore/io/xml_binary/core.py @@ -5,12 +5,11 @@ from typing import Literal from xml.etree.ElementTree import ParseError -import numpy as np from pydantic import ValidationError import dascore as dc from dascore.io import FiberIO, ScanPayload -from dascore.models import UTF8Str +from dascore.models import FiniteFloat, UTF8Str from dascore.utils.paths import coerce_to_upath from .utils import _load_patches, _paths_to_scan_patches, _read_xml_metadata @@ -19,8 +18,8 @@ class BinaryPatchAttrs(dc.PatchAttrs): """Patch attrs for Binary.""" - pulse_width: float = np.nan - gauge_length: float = np.nan + pulse_width: FiniteFloat | None = None + gauge_length: FiniteFloat | None = None zone_name: UTF8Str = "" diff --git a/dascore/models/__init__.py b/dascore/models/__init__.py index af0fd5a92..798c90677 100644 --- a/dascore/models/__init__.py +++ b/dascore/models/__init__.py @@ -21,6 +21,7 @@ CommaSeparatedStr, DateTime64, DTypeLike, + FiniteFloat, FrozenDictType, PositiveFiniteFloat, PositiveInt, @@ -37,6 +38,7 @@ "DTypeLike", "DascoreBaseModel", "DateTime64", + "FiniteFloat", "FrozenDictType", "InventoryModel", "PositiveFiniteFloat", diff --git a/dascore/models/types.py b/dascore/models/types.py index abbbb9ae8..4dd8085d2 100644 --- a/dascore/models/types.py +++ b/dascore/models/types.py @@ -95,5 +95,10 @@ def _to_unit_quantity(value): # A positive (> 0) integer. PositiveInt = Annotated[int, Field(gt=0)] +# A float which must be finite; nan/inf silently poison downstream math. +# Spell an optional number `FiniteFloat | None`: nan has no JSON spelling, +# so a nan-defaulted float writes `null` and then refuses to read it back. +FiniteFloat = Annotated[float, Field(allow_inf_nan=False)] + # A positive (> 0), finite (no nan/inf) float. PositiveFiniteFloat = Annotated[float, Field(gt=0, allow_inf_nan=False)] diff --git a/dascore/utils/models.py b/dascore/utils/models.py index 2c0186bec..3a85217e5 100644 --- a/dascore/utils/models.py +++ b/dascore/utils/models.py @@ -22,6 +22,7 @@ CommaSeparatedStr, DateTime64, DTypeLike, + FiniteFloat, FrozenDictType, PositiveFiniteFloat, PositiveInt, @@ -39,6 +40,7 @@ "DTypeLike", "DascoreBaseModel", "DateTime64", + "FiniteFloat", "FrozenDictType", "InventoryModel", "PositiveFiniteFloat", diff --git a/tests/test_models.py b/tests/test_models.py index a5a149f73..402fcfde0 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -8,6 +8,8 @@ import pytest from pydantic import Field, ValidationError +from dascore.core.attrs import PatchAttrs +from dascore.io.core import FiberIO from dascore.models import ( DascoreBaseModel, DateTime64, @@ -161,3 +163,67 @@ def test_unhashable_field_still_refuses(self): """A model holding an array is unhashable, and says so.""" with pytest.raises(TypeError, match="unhashable"): hash(_TestModel(array=np.arange(10))) + + +def _dascore_patch_attrs_classes(): + """Every PatchAttrs class DASCore itself declares, base included.""" + # The subclasses only exist once their io modules are imported, and other + # test modules register their own subclasses globally on import, so the + # walk both forces the load and keeps to DASCore's own classes. + FiberIO.manager.load_plugins() + found: dict[str, type[PatchAttrs]] = {} + stack = [PatchAttrs] + while stack: + cls = stack.pop() + stack.extend(cls.__subclasses__()) + if cls.__module__.startswith("dascore."): + found[cls.__name__] = cls + return [found[name] for name in sorted(found)] + + +# A required field has no default to round trip, so the walk states one. A +# new required field fails the assert below rather than dropping out of it. +_REQUIRED_ATTR_VALUES = {"gauge_length": 10.0} + + +def _minimal_attrs(cls): + """Build the emptiest legal instance of a PatchAttrs class.""" + required = {name for name, f in cls.model_fields.items() if f.is_required()} + assert not (unknown := required - set(_REQUIRED_ATTR_VALUES)), ( + f"{cls.__name__} requires {sorted(unknown)}, which " + "_REQUIRED_ATTR_VALUES does not state a value for." + ) + return cls(**{name: _REQUIRED_ATTR_VALUES[name] for name in required}) + + +@pytest.mark.parametrize( + "attrs_class", + _dascore_patch_attrs_classes(), + ids=lambda cls: cls.__name__, +) +class TestPatchAttrsSerialization: + """ + Every PatchAttrs class must survive a text round trip. + + Parametrized over the class walk rather than a list, so a format added + later is covered without touching this file. + """ + + def test_json_round_trip(self, attrs_class): + """A defaulted instance reconstructs from its own json.""" + attrs = _minimal_attrs(attrs_class) + out = attrs_class.model_validate_json(attrs.model_dump_json()) + assert out == attrs + + def test_no_field_defaults_to_a_non_finite_number(self, attrs_class): + """ + Nan and inf have no json spelling, so a float defaulting to one + writes null and then refuses to read it back. Optional numbers are + spelled `FiniteFloat | None`. + """ + for name, field in attrs_class.model_fields.items(): + default = field.get_default(call_default_factory=True) + if isinstance(default, float): + assert np.isfinite(default), ( + f"{attrs_class.__name__}.{name} defaults to {default}." + ) From b979fc3d58d8eaa4344d7a1db797fa6145c08080 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Thu, 13 Aug 2026 12:34:53 +0200 Subject: [PATCH 4/8] Let a serialized document name the class it holds A model did not say what it was when serialized, so a document could only be read by something which already knew. Every model now writes an object_type naming a registered class, and reads one back, so a custom PatchAttrs survives a round trip and a standalone object can be read on its own. The tag is written in text serializations only: a python-mode dump is what equality compares, what new() reconstructs from and what the spool index ingests, none of which want a key that is not a field. The five resource models keep their own object_type field, which pydantic needs to pick a class before an object exists; renaming it from 'type' is what lets the base class recognize and leave them alone, and the loader stops popping what every model now reads for itself. --- dascore/core/inventory.py | 40 ++-- dascore/core/inventory_loader.py | 29 +-- dascore/exceptions.py | 4 + dascore/models/base.py | 52 +++++ dascore/models/registry.py | 203 ++++++++++++++++++ tests/test_core/test_inventory.py | 16 +- tests/test_core/test_inventory_loader.py | 259 ++++++++++++++--------- tests/test_models.py | 195 +++++++++++++++++ 8 files changed, 653 insertions(+), 145 deletions(-) create mode 100644 dascore/models/registry.py diff --git a/dascore/core/inventory.py b/dascore/core/inventory.py index 80b4bafbb..c5e8e67a5 100644 --- a/dascore/core/inventory.py +++ b/dascore/core/inventory.py @@ -128,14 +128,16 @@ def _annotation_value(value): ] -def _type_tag(name: str): +def _object_type_tag(name: str): """ - Return the serialization-only type tag field for a tagged model class. - - The tag drives union dispatch and the authoring format's type - declaration in serialized YAML/JSON. Users never set it: it defaults to - the class name, the Literal annotation rejects any other value, and it - is hidden from repr. + Return the serialization-only ``object_type`` field of a union member. + + Every model states its class when serialized (see + [dascore.models.registry](`dascore.models.registry`)), but pydantic must + pick a class for these before an object exists, so the models sharing a + union declare the tag as a real field and the base class leaves them + alone. Users never set it: it defaults to the class name, the Literal + annotation rejects any other value, and it is hidden from repr. """ return Field(default=name, repr=False) @@ -253,7 +255,7 @@ def axis_index(self, label: str) -> int: class ExternalResource(InventoryModel): """External resource identified but not otherwise modeled by DASCore.""" - type: Literal["ExternalResource"] = _type_tag("ExternalResource") + object_type: Literal["ExternalResource"] = _object_type_tag("ExternalResource") resource_id: ResourceIdStr uri: str = Field(default="", description="URI or identifier for the resource.") name: str = Field(default="", description="Human-readable resource name.") @@ -269,7 +271,7 @@ class OpticalMeasurement(InventoryModel): datasheet claim is a legitimate record (method="datasheet"). """ - type: Literal["OpticalMeasurement"] = _type_tag("OpticalMeasurement") + object_type: Literal["OpticalMeasurement"] = _object_type_tag("OpticalMeasurement") resource_id: ResourceIdStr name: str = Field(default="", description="Human-readable measurement name.") method: str = Field( @@ -300,7 +302,7 @@ class OpticalMeasurement(InventoryModel): class Interrogator(InventoryModel): """DFOS interrogator unit used for data collection.""" - type: Literal["Interrogator"] = _type_tag("Interrogator") + object_type: Literal["Interrogator"] = _object_type_tag("Interrogator") resource_id: ResourceIdStr name: str = Field(default="", description="Human-readable resource name.") manufacturer: str = Field(default="", description="Manufacturer name.") @@ -314,7 +316,7 @@ class Interrogator(InventoryModel): class Enclosure(InventoryModel): """Physical housing, pipe, duct, conduit, or carrier resource.""" - type: Literal["Enclosure"] = _type_tag("Enclosure") + object_type: Literal["Enclosure"] = _object_type_tag("Enclosure") resource_id: ResourceIdStr name: str = Field(default="", description="Human-readable resource name.") enclosure_type: str = Field( @@ -339,7 +341,7 @@ class Enclosure(InventoryModel): class Cable(InventoryModel): """Physical cable containing one or more fiber segments.""" - type: Literal["Cable"] = _type_tag("Cable") + object_type: Literal["Cable"] = _object_type_tag("Cable") resource_id: ResourceIdStr name: str = Field(default="", description="Human-readable resource name.") manufacturer: str = Field(default="", description="Manufacturer name.") @@ -360,7 +362,7 @@ class Cable(InventoryModel): _Resource: TypeAlias = Annotated[ Interrogator | Cable | Enclosure | ExternalResource | OpticalMeasurement, - Field(discriminator="type"), + Field(discriminator="object_type"), ] @@ -431,7 +433,7 @@ def _check_measurement_pairing(self) -> Self: class FiberSegment(_OpticalComponentBase): """Length of optical fiber within a cable, patch cord, or other run.""" - type: Literal["FiberSegment"] = _type_tag("FiberSegment") + object_type: Literal["FiberSegment"] = _object_type_tag("FiberSegment") container: Cable | str | None = Field( default=None, description="Cable containing this fiber." ) @@ -470,7 +472,7 @@ def attenuation_db_per_km(self) -> float | tuple[float, ...] | None: class Connector(_OpticalComponentBase): """Optical connector in an optical path.""" - type: Literal["Connector"] = _type_tag("Connector") + object_type: Literal["Connector"] = _object_type_tag("Connector") container: Enclosure | str | None = Field( default=None, description="Enclosure housing this connector." ) @@ -480,7 +482,7 @@ class Connector(_OpticalComponentBase): class Splice(_OpticalComponentBase): """Optical splice in an optical path.""" - type: Literal["Splice"] = _type_tag("Splice") + object_type: Literal["Splice"] = _object_type_tag("Splice") container: Enclosure | str | None = Field( default=None, description="Enclosure housing this splice." ) @@ -490,7 +492,7 @@ class Splice(_OpticalComponentBase): class Terminator(_OpticalComponentBase): """Optical path terminator.""" - type: Literal["Terminator"] = _type_tag("Terminator") + object_type: Literal["Terminator"] = _object_type_tag("Terminator") container: Enclosure | str | None = Field( default=None, description="Enclosure housing this terminator." ) @@ -501,7 +503,7 @@ class Terminator(_OpticalComponentBase): OpticalComponent: TypeAlias = Annotated[ FiberSegment | Connector | Splice | Terminator, - Field(discriminator="type"), + Field(discriminator="object_type"), ] @@ -1686,7 +1688,7 @@ class InventoryNames(NamedTuple): # Fields which name or tag a record rather than describe it: the type # discriminator and resource id a shareable record carries, and the codes # locating an acquisition (a patch's acquisition_key already states them). -_IDENTITY_FIELDS = frozenset({"type", "resource_id", "code", "location_code"}) +_IDENTITY_FIELDS = frozenset({"object_type", "resource_id", "code", "location_code"}) # What a distance-ranged track carries to place itself, which is its extent # rather than a value the channels inside it take. diff --git a/dascore/core/inventory_loader.py b/dascore/core/inventory_loader.py index b61040207..33b26c705 100644 --- a/dascore/core/inventory_loader.py +++ b/dascore/core/inventory_loader.py @@ -13,7 +13,7 @@ skipped, so a directory cannot load as an entity silently missing the tracks its own files state. -The contract, in one line: **file declares type, container agrees, name +The contract, in one line: **file declares object_type, container agrees, name implies identity, envelope implies version.** Every object file states what it is, its container checks that statement rather than supplying it, its name decides which entity it is, and the top-level ``inventory.yaml`` @@ -57,6 +57,7 @@ MissingOptionalDependencyError, ) from dascore.models import InventoryModel, TimeRangedModel +from dascore.models.registry import TAG_FIELD from dascore.utils.misc import check_code, optional_import from dascore.utils.time import to_datetime64 @@ -229,7 +230,7 @@ def _declared_type(path: Path) -> str | None: data = _read_object(path) except (InvalidInventoryError, MissingOptionalDependencyError): return None - declared = data.get("type") + declared = data.get(TAG_FIELD) return declared if isinstance(declared, str) else None @@ -317,27 +318,26 @@ def _pick_model(data: dict, container: _Container, source: Path): The container never supplies the type: a file which does not say what it is has not participated in the format, and one which says something its container cannot hold is misfiled rather than reinterpreted. + + The tag is left in the data. Every model reads its own, so the file's + statement is checked a second time by the object it builds. """ - declared = data.get("type") + declared = data.get(TAG_FIELD) legal = tuple(x.__name__ for x in container.models) # Not merely absent: a type which is not a name at all -- a list, a # mapping -- names no model either, and must not reach the lookups below. if not isinstance(declared, str): msg = ( - f"{_quote(source)} declares no type. Every object file states " - f"what it is, e.g. 'type: {legal[0]}'." + f"{_quote(source)} declares no {TAG_FIELD}. Every object file " + f"states what it is, e.g. '{TAG_FIELD}: {legal[0]}'." ) raise InvalidInventoryError(msg) for model in container.models: if model.__name__ == declared: - # The tag is a real, discriminating field only on the models - # which share a union; everywhere else it belongs to the format. - if "type" not in model.model_fields: - data.pop("type") return model known = "an inventory model" if declared in _model_names() else "unknown" msg = ( - f"{_quote(source)} declares type {declared!r} ({known}), which " + f"{_quote(source)} declares {TAG_FIELD} {declared!r} ({known}), which " f"{source.parent.name} cannot hold. Expected one of {legal}." ) raise InvalidInventoryError(msg) @@ -793,10 +793,10 @@ def _load_envelope(root: Path) -> dict[str, Any] | None: raise InvalidInventoryError(msg) source = found[0] data = _read_object(source) - if (declared := data.pop("type", None)) != Inventory.__name__: + if (declared := data.get(TAG_FIELD)) != Inventory.__name__: msg = ( - f"{_quote(source)} declares type {declared!r}; the envelope " - f"declares 'type: {Inventory.__name__}'." + f"{_quote(source)} declares {TAG_FIELD} {declared!r}; the envelope " + f"declares '{TAG_FIELD}: {Inventory.__name__}'." ) raise InvalidInventoryError(msg) _refuse_supplied(data, _ENVELOPE_COLLECTIONS, source, "the envelope") @@ -837,7 +837,8 @@ def check(path: Path): elif _object_suffix(path) is not None: if (declared := _declared_type(path)) in known: msg = ( - f"{path.relative_to(root)} declares type {declared!r} but " + f"{path.relative_to(root)} declares {TAG_FIELD} " + f"{declared!r} but " "nothing contains it. An inventory object lives in one of " f"{tuple(_CONTAINERS)}, named for the entity it is." ) diff --git a/dascore/exceptions.py b/dascore/exceptions.py index c158b0f3a..df9f90301 100644 --- a/dascore/exceptions.py +++ b/dascore/exceptions.py @@ -171,3 +171,7 @@ class RemoteCacheError(IOError, DASCoreError): class InvalidInventoryError(ValueError, DASCoreError): """Raised when inventory metadata violates the DASDAE inventory model.""" + + +class InvalidModelTagError(ValueError, DASCoreError): + """Raised when a serialized document names its model class illegally.""" diff --git a/dascore/models/base.py b/dascore/models/base.py index d516b35df..4f3e00bf7 100644 --- a/dascore/models/base.py +++ b/dascore/models/base.py @@ -4,6 +4,7 @@ from collections.abc import Mapping from functools import cached_property +from typing import Any import numpy as np import pandas as pd @@ -11,12 +12,21 @@ BaseModel, ConfigDict, Field, + SerializationInfo, + SerializerFunctionWrapHandler, + model_serializer, model_validator, ) from typing_extensions import Self from dascore.compat import is_array_like from dascore.exceptions import InvalidInventoryError +from dascore.models.registry import ( + TAG_FIELD, + check_tag_matches, + get_model_tag, + register_model, +) from dascore.models.types import DateTime64, FrozenDictType from dascore.utils.misc import _all_null, all_close from dascore.utils.time import to_datetime64 @@ -100,6 +110,48 @@ class DascoreBaseModel(BaseModel): arbitrary_types_allowed=True, ) + def __init_subclass__(cls, **kwargs): + """Register every model so a document can name it.""" + super().__init_subclass__(**kwargs) + register_model(cls) + + @model_serializer(mode="wrap") + def _write_object_type( + self, handler: SerializerFunctionWrapHandler, info: SerializationInfo + ) -> Any: + """ + Name this class in the document, in text serializations only. + + Only json mode, because a python-mode dump is not a document: it is + what equality compares, what ``new`` reconstructs from and what the + index ingests, none of which want a key that is not a field. + + Models which discriminate a union declare ``object_type`` as a real + field, and pydantic has already written it. + """ + out = handler(self) + if info.mode == "json" and TAG_FIELD not in type(self).model_fields: + out[TAG_FIELD] = get_model_tag(type(self)) + return out + + @model_validator(mode="before") + @classmethod + def _read_object_type(cls, data: Any) -> Any: + """ + Consume a document's class tag, refusing one which names another class. + + The tag is never required: a document dispatches on it before it gets + here, and a hand-written object or a nested one may simply not state + it. What it may not do is disagree. + """ + if not isinstance(data, Mapping) or TAG_FIELD in cls.model_fields: + return data + if TAG_FIELD not in data: + return data + data = dict(data) + check_tag_matches(cls, data.pop(TAG_FIELD)) + return data + def new(self, **kwargs) -> Self: """Create new instance with some attributed updated.""" out = self.model_dump(exclude_unset=True) diff --git a/dascore/models/registry.py b/dascore/models/registry.py new file mode 100644 index 000000000..b9ff54100 --- /dev/null +++ b/dascore/models/registry.py @@ -0,0 +1,203 @@ +""" +The registry which lets a serialized document name the class it holds. + +A document states its class in an ``object_type`` key holding a registered +name, never an import path. A dotted path would weld stored documents to +today's module layout and make reading one an arbitrary-import surface; +a registered name costs the same to write and has neither property. +""" + +from __future__ import annotations + +import re +import warnings +from collections.abc import Mapping + +from dascore.exceptions import InvalidModelTagError +from dascore.utils.plugins import get_entry_point_loaders + +# The key a document states its class in. Specific enough that no reader's +# header and no user's extra attribute is expected to spell it, which is why +# the validator may consume it wherever it appears. +TAG_FIELD = "object_type" + +NAMESPACE_SEP = ":" + +# DASCore's own models are registered bare, so a hand-authored file says +# `object_type: Cable` and a plugin's says `object_type: myplugin:Square`. +DASCORE_NAMESPACE = "dascore" + +FIBER_IO_GROUP = "dascore.fiber_io" + +# `[namespace:]ClassName[-x.y.z]`. A python class name holds neither ":" +# nor "-", so the three parts can never be read for one another. Nothing +# reads a version today; the grammar only keeps room for one, where an +# absent version will mean the earliest. +TAG_PATTERN = re.compile(r"^(?:[a-z_][\w.]*:)?[A-Za-z_]\w*(?:-\d+\.\d+\.\d+)?$") + +_REGISTRY: dict[str, type] = {} + +# Set once the io plugins have been swept looking for an unresolved tag. +_plugins_swept = False + + +def _derive_namespace(cls: type) -> str: + """Return the namespace a class registers under.""" + # Derived rather than declared: it makes a plugin's models namespaced + # with no ceremony, and leaves no way to squat a bare name. + return cls.__module__.split(".", 1)[0] + + +def get_model_tag(cls: type) -> str: + """ + Return the tag which names a model class in a document. + + An out-of-tree class is namespaced by the package which declares it; + DASCore's own classes are bare. + """ + namespace = _derive_namespace(cls) + if namespace == DASCORE_NAMESPACE: + return cls.__name__ + return f"{namespace}{NAMESPACE_SEP}{cls.__name__}" + + +def register_model(cls: type) -> None: + """ + Add a model class to the registry under its derived tag. + + Classes declared inside a function are skipped: nothing can resolve a + name which only exists while its enclosing call runs, and two of them + sharing a name is neither a mistake nor resolvable. + """ + if "" in cls.__qualname__: + return + tag = get_model_tag(cls) + existing = _REGISTRY.get(tag) + if existing is not None and _identity(existing) != _identity(cls): + _report_collision(tag, existing, cls) + # A module re-imported under the same name replaces its own entry. + _REGISTRY[tag] = cls + + +def _identity(cls: type) -> tuple[str, str]: + """Return what makes a class the same class across a re-import.""" + return (cls.__module__, cls.__qualname__) + + +def _report_collision(tag: str, existing: type, new: type) -> None: + """Complain that two different classes want one tag.""" + msg = ( + f"Two models claim the tag {tag!r}: {existing.__module__}." + f"{existing.__qualname__} and {new.__module__}.{new.__qualname__}. " + "A tag must name one class; rename one of them." + ) + # DASCore's own names are its own to keep unique, and a test pins it. + # Out of tree the collision may be between two packages a user merely + # installed, which they cannot fix by renaming, so it warns and the + # last registration wins -- as duplicate entry points already do. + if _derive_namespace(new) == DASCORE_NAMESPACE: + raise InvalidModelTagError(msg) + warnings.warn(msg, UserWarning, stacklevel=2) + + +def _sweep_plugin_modules() -> None: + """Import the io plugins, defining any models they declare.""" + global _plugins_swept + if _plugins_swept: + return + _plugins_swept = True + for loader in get_entry_point_loaders(FIBER_IO_GROUP).values(): + try: + loader() + except Exception: + # A plugin which cannot be imported has no models to find. It is + # not reported here: FiberIO warns about the same plugin when it + # loads formats, and a failure to resolve one tag is not the + # place to announce an unrelated broken install. + continue + + +def resolve_model_tag(tag: str) -> type | None: + """ + Return the class a tag names, or None if nothing registers it. + + Raises if the tag is not a legal tag at all, which is a malformed + document rather than an unknown class. + """ + if not isinstance(tag, str) or not TAG_PATTERN.match(tag): + msg = ( + f"{tag!r} is not a legal {TAG_FIELD}. Expected a registered name, " + "optionally namespaced, eg 'Cable' or 'myplugin:Square'." + ) + raise InvalidModelTagError(msg) + if (cls := _REGISTRY.get(tag)) is not None: + return cls + # A format's models only exist once its module is imported, and io + # modules are imported lazily, so an unknown name is worth one sweep. + _sweep_plugin_modules() + return _REGISTRY.get(tag) + + +def check_tag_matches(cls: type, tag: str) -> None: + """ + Refuse a document whose tag names a class the one being built is not. + + A tag naming a subclass is accepted: such a document holds everything + the class being built declares, which is what a caller asking for the + base class asked for. An unregistered tag is accepted too -- the caller + named the class, so there is nothing for the document to disagree with. + """ + declared = resolve_model_tag(tag) + if declared is None or issubclass(declared, cls): + return + msg = ( + f"A document declaring {TAG_FIELD} {tag!r} cannot be read as " + f"{cls.__name__}: {declared.__name__} is not one." + ) + raise InvalidModelTagError(msg) + + +def resolve_tagged_model( + data: Mapping, + default: type | None = None, + source: str | None = None, +) -> type: + """ + Return the model class a document names. + + Parameters + ---------- + data + The document, which states its class in its ``object_type`` key. + default + The class to fall back on when the document names one which is not + registered, usually because a plugin which wrote it is not + installed. Without one, an unresolved name raises. + source + Where the document came from, used in messages. + """ + tag = data.get(TAG_FIELD) if isinstance(data, Mapping) else None + where = f" in {source}" if source else "" + if tag is None: + if default is None: + msg = ( + f"The document{where} declares no {TAG_FIELD}, and nothing " + "else says which class it holds." + ) + raise InvalidModelTagError(msg) + return default + if (cls := resolve_model_tag(tag)) is not None: + return cls + msg = ( + f"Nothing registers the {TAG_FIELD} {tag!r}{where}. It was likely " + "written by a package which is not installed." + ) + if default is None: + raise InvalidModelTagError(msg) + warnings.warn(f"{msg} Reading it as {default.__name__}.", UserWarning) + return default + + +def registered_models() -> dict[str, type]: + """Return the registered classes, keyed by the tag naming each.""" + return dict(_REGISTRY) diff --git a/tests/test_core/test_inventory.py b/tests/test_core/test_inventory.py index fd24754b2..f4df337b2 100644 --- a/tests/test_core/test_inventory.py +++ b/tests/test_core/test_inventory.py @@ -874,7 +874,7 @@ def test_keyless_dict_resource_adopts_key(self): """A dict resource without resource_id adopts its pool key.""" pytest.importorskip("yaml") inventory = inv.Inventory( - resources={"cab-1": {"type": "Cable", "name": "mycable"}} + resources={"cab-1": {"object_type": "Cable", "name": "mycable"}} ) assert inventory.get_resource("cab-1").resource_id == "cab-1" loaded = inv.Inventory.from_yaml(inventory.to_yaml()) @@ -1130,22 +1130,22 @@ def test_values_equal_branches(self): assert values_equal((1.0, np.nan), (1.0, np.nan)) -class TestTypeTag: - """The serialization type tag is invisible to users.""" +class TestObjectTypeTag: + """The union members' own tag field is invisible to users.""" def test_hidden_from_repr(self): """Hidden from repr.""" cable = inv.Cable(resource_id="c1", name="c") - assert "type" not in repr(cable) + assert "object_type" not in repr(cable) def test_present_in_dump(self): - """Present in dump.""" - assert inv.Cable(resource_id="c1").model_dump()["type"] == "Cable" + """Present in dump, unlike the tag every other model is given.""" + assert inv.Cable(resource_id="c1").model_dump()["object_type"] == "Cable" def test_wrong_tag_rejected(self): """Wrong tag rejected.""" with pytest.raises(ValidationError): - inv.Cable(resource_id="c1", type="Enclosure") + inv.Cable(resource_id="c1", object_type="Enclosure") class TestUniformAttachments: @@ -1241,7 +1241,7 @@ def test_frozen_record_disagreeing_with_its_key_is_refused(self): """A record is read as a record whatever kind of mapping it is.""" # Read as an object instead, its resource_id goes unseen and the # mismatch only surfaces on the next load. - record = FrozenDict({"type": "Cable", "resource_id": "elsewhere"}) + record = FrozenDict({"object_type": "Cable", "resource_id": "elsewhere"}) with pytest.raises(ValidationError, match="disagrees with resource_id"): inv.Inventory(resources={"cable-01": record}) diff --git a/tests/test_core/test_inventory_loader.py b/tests/test_core/test_inventory_loader.py index 4a5c530b8..c159d21fc 100644 --- a/tests/test_core/test_inventory_loader.py +++ b/tests/test_core/test_inventory_loader.py @@ -8,6 +8,7 @@ import numpy as np import pytest +from pydantic import ValidationError import dascore as dc from dascore.core import inventory as inv @@ -17,13 +18,14 @@ MissingOptionalDependencyError, ) from dascore.models import InventoryModel, TimeRangedModel +from dascore.models.registry import TAG_FIELD pytest.importorskip("yaml") # A minimal directory which loads: one acquisition names everything above it. MINIMAL = { - "acquisitions/DAS.L001..RAW.yaml": "type: Acquisition\ndata_category: DAS\n", + "acquisitions/DAS.L001..RAW.yaml": "object_type: Acquisition\ndata_category: DAS\n", } @@ -116,7 +118,7 @@ def test_resolves_like_any_inventory(self, make_inventory): out = make_inventory( { "acquisitions/DAS.L001..RAW.yaml": ( - "type: Acquisition\ndata_category: DAS\ngauge_length: 10.0\n" + "object_type: Acquisition\ndata_category: DAS\ngauge_length: 10.0\n" ), } ) @@ -126,15 +128,16 @@ def test_full_directory(self, make_inventory): """Every container contributes to one inventory.""" out = make_inventory( { - "inventory.yaml": "type: Inventory\nschema_version: 1\n", + "inventory.yaml": "object_type: Inventory\nschema_version: 1\n", "resources/int_01.yaml": ( - "type: Interrogator\nmanufacturer: Fake\nmodel: FI-1\n" + "object_type: Interrogator\nmanufacturer: Fake\nmodel: FI-1\n" ), - "networks/DAS.yaml": "type: Network\nname: test network\n", - "fiber_arrays/DAS.L001.yaml": "type: FiberArray\nname: first\n", - "stations/DAS.STA1.yaml": "type: Station\nname: a station\n", + "networks/DAS.yaml": "object_type: Network\nname: test network\n", + "fiber_arrays/DAS.L001.yaml": "object_type: FiberArray\nname: first\n", + "stations/DAS.STA1.yaml": "object_type: Station\nname: a station\n", "acquisitions/DAS.L001..RAW.yaml": ( - "type: Acquisition\ndata_category: DAS\ninterrogator: int_01\n" + "object_type: Acquisition\ndata_category: DAS\n" + "interrogator: int_01\n" ), } ) @@ -152,7 +155,8 @@ def test_json_and_yaml_are_interchangeable(self, tmp_path): tmp_path / "yaml_form", { "acquisitions/DAS.L001..RAW.yaml": ( - "type: Acquisition\ndata_category: DAS\ngauge_length: 4.0\n" + "object_type: Acquisition\ndata_category: DAS\n" + "gauge_length: 4.0\n" ) }, ) @@ -162,7 +166,7 @@ def test_json_and_yaml_are_interchangeable(self, tmp_path): tmp_path / "json_form", { "acquisitions/DAS.L001..RAW.json": ( - '{"type": "Acquisition", "data_category": "DAS", ' + '{"object_type": "Acquisition", "data_category": "DAS", ' '"gauge_length": 4.0}' ) }, @@ -172,7 +176,9 @@ def test_json_and_yaml_are_interchangeable(self, tmp_path): def test_yml_suffix(self, make_inventory): """The short YAML suffix is the same spelling.""" - out = make_inventory({"acquisitions/DAS.L001..RAW.yml": "type: Acquisition\n"}) + out = make_inventory( + {"acquisitions/DAS.L001..RAW.yml": "object_type: Acquisition\n"} + ) assert out.networks[0].fiber_arrays[0].acquisitions[0].code == "RAW" def test_envelope_is_optional(self, make_inventory): @@ -186,7 +192,7 @@ def test_envelope_states_the_singletons(self, make_inventory): { **MINIMAL, "inventory.yaml": ( - "type: Inventory\n" + "object_type: Inventory\n" "resource_id: my-inventory\n" "coordinate_reference_system:\n" " authority: EPSG\n" @@ -204,7 +210,7 @@ def test_entity_directory_form(self, make_inventory): { **MINIMAL, "fiber_arrays/DAS.L001/attrs.yaml": ( - "type: FiberArray\nname: from a directory\n" + "object_type: FiberArray\nname: from a directory\n" ), } ) @@ -221,7 +227,7 @@ def test_non_participating_files_are_ignored(self, make_inventory): "notes/scratch.yaml": "just: a note\n", # Nor is one which does not parse at all. "notes/broken.yaml": "{[unclosed\n", - ".hidden/DAS.yaml": "type: Network\n", + ".hidden/DAS.yaml": "object_type: Network\n", "acquisitions/README.txt": "a note beside the objects\n", } ) @@ -232,7 +238,7 @@ def test_resource_id_may_hold_dots(self, make_inventory): out = make_inventory( { **MINIMAL, - "resources/cable.01.yaml": "type: Cable\nname: a cable\n", + "resources/cable.01.yaml": "object_type: Cable\nname: a cable\n", } ) assert out.get_resource("cable.01").name == "a cable" @@ -242,7 +248,7 @@ def test_restated_address_may_agree(self, make_inventory): out = make_inventory( { "acquisitions/DAS.L001.01.RAW.yaml": ( - "type: Acquisition\ncode: RAW\nlocation_code: '01'\n" + "object_type: Acquisition\ncode: RAW\nlocation_code: '01'\n" ) } ) @@ -256,7 +262,7 @@ class TestEpochNames: def test_date_only_means_midnight_utc(self, make_inventory): """A date-only suffix is valid when that precision suffices.""" out = make_inventory( - {"acquisitions/DAS.L001..RAW@2024-06-01.yaml": "type: Acquisition\n"} + {"acquisitions/DAS.L001..RAW@2024-06-01.yaml": "object_type: Acquisition\n"} ) acquisition = out.networks[0].fiber_arrays[0].acquisitions[0] assert acquisition.start_time == np.datetime64("2024-06-01T00:00:00", "ns") @@ -266,7 +272,7 @@ def test_basic_time_and_fractional_seconds(self, make_inventory): out = make_inventory( { "acquisitions/DAS.L001..RAW@2024-05-12T103000.12.yaml": ( - "type: Acquisition\n" + "object_type: Acquisition\n" ) } ) @@ -279,7 +285,7 @@ def test_suffix_agrees_with_stated_start(self, make_inventory): out = make_inventory( { "acquisitions/DAS.L001..RAW@2024-06-01.yaml": ( - "type: Acquisition\nstart_time: '2024-06-01'\n" + "object_type: Acquisition\nstart_time: '2024-06-01'\n" ) } ) @@ -291,10 +297,11 @@ def test_epochs_of_one_acquisition(self, make_inventory): out = make_inventory( { "acquisitions/DAS.L001..RAW.yaml": ( - "type: Acquisition\nend_time: '2024-06-01'\ngauge_length: 10.0\n" + "object_type: Acquisition\nend_time: '2024-06-01'\n" + "gauge_length: 10.0\n" ), "acquisitions/DAS.L001..RAW@2024-06-01.yaml": ( - "type: Acquisition\ngauge_length: 5.0\n" + "object_type: Acquisition\ngauge_length: 5.0\n" ), } ) @@ -308,12 +315,14 @@ def test_acquisition_lands_in_its_array_epoch(self, make_inventory): out = make_inventory( { "fiber_arrays/DAS.L001.yaml": ( - "type: FiberArray\nname: first\nend_time: '2024-06-01'\n" + "object_type: FiberArray\nname: first\nend_time: '2024-06-01'\n" ), "fiber_arrays/DAS.L001@2024-06-01.yaml": ( - "type: FiberArray\nname: second\n" + "object_type: FiberArray\nname: second\n" + ), + "acquisitions/DAS.L001..RAW@2024-07-01.yaml": ( + "object_type: Acquisition\n" ), - "acquisitions/DAS.L001..RAW@2024-07-01.yaml": "type: Acquisition\n", } ) arrays = {x.name: x for x in out.networks[0].fiber_arrays} @@ -326,37 +335,39 @@ class TestNearMisses: def test_typo_in_container_name(self, make_inventory): """A typo must not quietly load an inventory with no acquisitions.""" - files = {"aquisitions/DAS.L001..RAW.yaml": "type: Acquisition\n"} + files = {"aquisitions/DAS.L001..RAW.yaml": "object_type: Acquisition\n"} with pytest.raises(InvalidInventoryError, match="nothing contains it"): make_inventory(files) def test_model_file_at_the_root(self, make_inventory): """An object at the root is outside every container.""" - files = {**MINIMAL, "DAS.L001.yaml": "type: FiberArray\n"} + files = {**MINIMAL, "DAS.L001.yaml": "object_type: FiberArray\n"} with pytest.raises(InvalidInventoryError, match="nothing contains it"): make_inventory(files) def test_missing_type(self, make_inventory): """A file which does not say what it is has not participated.""" files = {"acquisitions/DAS.L001..RAW.yaml": "data_category: DAS\n"} - with pytest.raises(InvalidInventoryError, match="declares no type"): + with pytest.raises(InvalidInventoryError, match="declares no object_type"): make_inventory(files) def test_wrong_container(self, make_inventory): """The container checks the declared type rather than supplying it.""" - files = {"fiber_arrays/DAS.L001.yaml": "type: Acquisition\n"} + files = {"fiber_arrays/DAS.L001.yaml": "object_type: Acquisition\n"} with pytest.raises(InvalidInventoryError, match="cannot hold"): make_inventory(files) def test_unknown_type(self, make_inventory): """A type which names no model is unknown rather than misfiled.""" - files = {"fiber_arrays/DAS.L001.yaml": "type: Telescope\n"} + files = {"fiber_arrays/DAS.L001.yaml": "object_type: Telescope\n"} with pytest.raises(InvalidInventoryError, match="unknown"): make_inventory(files) def test_restated_address_disagrees(self, make_inventory): """There is never a precedence rule between two spellings.""" - files = {"acquisitions/DAS.L001..RAW.yaml": "type: Acquisition\ncode: DEC\n"} + files = { + "acquisitions/DAS.L001..RAW.yaml": "object_type: Acquisition\ncode: DEC\n" + } with pytest.raises(InvalidInventoryError, match="must agree with the name"): make_inventory(files) @@ -364,7 +375,7 @@ def test_restated_start_time_disagrees(self, make_inventory): """The epoch suffix is a restated address, so it must agree.""" files = { "acquisitions/DAS.L001..RAW@2024-06-01.yaml": ( - "type: Acquisition\nstart_time: '2024-06-02'\n" + "object_type: Acquisition\nstart_time: '2024-06-02'\n" ) } with pytest.raises(InvalidInventoryError, match="must agree with the name"): @@ -378,7 +389,7 @@ def test_restated_start_time_disagrees(self, make_inventory): ) def test_illegal_address_token_names_the_file(self, make_inventory, name, level): """The entity a token names is built from every address, not one file.""" - files = {f"acquisitions/{name}": "type: Acquisition\n"} + files = {f"acquisitions/{name}": "object_type: Acquisition\n"} with pytest.raises(InvalidInventoryError, match=f"names {level}") as info: make_inventory(files) # The point of the check is which file to open, so assert the name. @@ -386,7 +397,7 @@ def test_illegal_address_token_names_the_file(self, make_inventory, name, level) def test_wrong_token_count(self, make_inventory): """An acquisition name is an address of four tokens.""" - files = {"acquisitions/DAS.L001.RAW.yaml": "type: Acquisition\n"} + files = {"acquisitions/DAS.L001.RAW.yaml": "object_type: Acquisition\n"} with pytest.raises(InvalidInventoryError, match="address of 4"): make_inventory(files) @@ -394,7 +405,7 @@ def test_schema_version_outside_the_envelope(self, make_inventory): """The envelope versions the document exactly once.""" files = { "acquisitions/DAS.L001..RAW.yaml": ( - "type: Acquisition\nschema_version: 1\n" + "object_type: Acquisition\nschema_version: 1\n" ) } with pytest.raises(InvalidInventoryError, match="envelope versions"): @@ -404,7 +415,7 @@ def test_invalid_field_names_the_file(self, make_inventory): """A model error says which file could not be read.""" files = { "acquisitions/DAS.L001..RAW.yaml": ( - "type: Acquisition\ngauge_length: not a number\n" + "object_type: Acquisition\ngauge_length: not a number\n" ) } with pytest.raises(InvalidInventoryError, match=r"DAS\.L001\.\.RAW\.yaml"): @@ -435,9 +446,9 @@ class TestOverlookedInput: def test_object_filed_inside_an_entity_directory(self, make_inventory): """An object one level too deep must not be silently dropped.""" files = { - "fiber_arrays/DAS.L001/attrs.yaml": "type: FiberArray\n", + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", "fiber_arrays/DAS.L001/misplaced/DAS.L001..RAW.yaml": ( - "type: Acquisition\n" + "object_type: Acquisition\n" ), } with pytest.raises(InvalidInventoryError, match="nothing contains it"): @@ -447,9 +458,9 @@ def test_object_filed_inside_an_entity_directory(self, make_inventory): def test_upper_case_suffixes(self, make_inventory, suffix): """A case-insensitive filesystem holds one file, not two spellings.""" text = ( - '{"type": "Acquisition"}' + '{"object_type": "Acquisition"}' if suffix == "JSON" - else "type: Acquisition\ndata_category: DAS\n" + else "object_type: Acquisition\ndata_category: DAS\n" ) out = make_inventory({f"acquisitions/DAS.L001..RAW.{suffix}": text}) assert out.networks[0].fiber_arrays[0].acquisitions[0].code == "RAW" @@ -457,7 +468,10 @@ def test_upper_case_suffixes(self, make_inventory, suffix): def test_upper_case_envelope(self, make_inventory): """The envelope is found however its suffix is spelled.""" out = make_inventory( - {**MINIMAL, "inventory.YAML": "type: Inventory\nresource_id: shouted\n"} + { + **MINIMAL, + "inventory.YAML": "object_type: Inventory\nresource_id: shouted\n", + } ) assert out.resource_id == "shouted" @@ -465,12 +479,12 @@ def test_child_outliving_its_parent_epoch(self, make_inventory): """Starting inside an epoch is not enough to belong to it.""" files = { "fiber_arrays/DAS.L001.yaml": ( - "type: FiberArray\nend_time: '2024-06-01'\n" + "object_type: FiberArray\nend_time: '2024-06-01'\n" ), - "fiber_arrays/DAS.L001@2024-06-01.yaml": "type: FiberArray\n", + "fiber_arrays/DAS.L001@2024-06-01.yaml": "object_type: FiberArray\n", # Starts inside the first epoch and never ends, so resolution # after June would find the second array, which does not hold it. - "acquisitions/DAS.L001..RAW@2024-05-01.yaml": "type: Acquisition\n", + "acquisitions/DAS.L001..RAW@2024-05-01.yaml": "object_type: Acquisition\n", } with pytest.raises(InvalidInventoryError, match="runs past"): make_inventory(files) @@ -480,11 +494,11 @@ def test_child_ending_exactly_at_the_boundary_fits(self, make_inventory): out = make_inventory( { "fiber_arrays/DAS.L001.yaml": ( - "type: FiberArray\nend_time: '2024-06-01'\n" + "object_type: FiberArray\nend_time: '2024-06-01'\n" ), - "fiber_arrays/DAS.L001@2024-06-01.yaml": "type: FiberArray\n", + "fiber_arrays/DAS.L001@2024-06-01.yaml": "object_type: FiberArray\n", "acquisitions/DAS.L001..RAW@2024-05-01.yaml": ( - "type: Acquisition\nend_time: '2024-06-01'\n" + "object_type: Acquisition\nend_time: '2024-06-01'\n" ), } ) @@ -493,7 +507,10 @@ def test_child_ending_exactly_at_the_boundary_fits(self, make_inventory): def test_unreadable_envelope_value_names_the_file(self, make_inventory): """An envelope error reads like every other error this format raises.""" - files = {**MINIMAL, "inventory.yaml": "type: Inventory\nschema_version: nope\n"} + files = { + **MINIMAL, + "inventory.yaml": "object_type: Inventory\nschema_version: nope\n", + } with pytest.raises(InvalidInventoryError, match="Could not read the envelope"): make_inventory(files) @@ -508,17 +525,17 @@ def test_unreadable_envelope_value_names_the_file(self, make_inventory): def test_nested_collections_are_refused(self, make_inventory, where, field, member): """A file may not state what the directory supplies and would replace.""" declared = "Network" if where.startswith("networks") else "FiberArray" - text = f"type: {declared}\n{field}:\n - {member}\n" + text = f"object_type: {declared}\n{field}:\n - {member}\n" with pytest.raises(InvalidInventoryError, match=field): make_inventory({**MINIMAL, where: text}) def test_child_predating_its_parent_epoch(self, make_inventory): """Containment is checked at the near end as well as the far one.""" files = { - "networks/DAS@2024-01-01.yaml": "type: Network\n", + "networks/DAS@2024-01-01.yaml": "object_type: Network\n", # No start of its own, so it claims the unbounded past, which is # outside a network beginning in 2024. - "stations/DAS.STA1.yaml": "type: Station\nend_time: '2020-01-01'\n", + "stations/DAS.STA1.yaml": "object_type: Station\nend_time: '2020-01-01'\n", } with pytest.raises(InvalidInventoryError, match="starts before"): make_inventory(files) @@ -527,11 +544,11 @@ def test_child_ending_after_its_parent_epoch(self, make_inventory): """The far end refuses a real end time, not only an unset one.""" files = { "fiber_arrays/DAS.L001.yaml": ( - "type: FiberArray\nend_time: '2024-06-01'\n" + "object_type: FiberArray\nend_time: '2024-06-01'\n" ), - "fiber_arrays/DAS.L001@2024-06-01.yaml": "type: FiberArray\n", + "fiber_arrays/DAS.L001@2024-06-01.yaml": "object_type: FiberArray\n", "acquisitions/DAS.L001..RAW@2024-05-01.yaml": ( - "type: Acquisition\nend_time: '2024-07-01'\n" + "object_type: Acquisition\nend_time: '2024-07-01'\n" ), } with pytest.raises(InvalidInventoryError, match="runs past"): @@ -549,7 +566,7 @@ def test_symlink_pointing_at_the_inventory(self, tmp_path): def test_envelope_stating_only_its_type(self, tmp_path): """An envelope says the directory is an inventory, whatever it holds.""" root = write_inventory( - tmp_path / "bare", {"inventory.yaml": "type: Inventory\n"} + tmp_path / "bare", {"inventory.yaml": "object_type: Inventory\n"} ) assert not dc.inventory(root).networks @@ -568,7 +585,7 @@ def no_yaml(name, **kwargs): root = write_inventory( tmp_path / "json_only", { - "acquisitions/DAS.L001..RAW.json": '{"type": "Acquisition"}', + "acquisitions/DAS.L001..RAW.json": '{"object_type": "Acquisition"}', # Field material the loader must step over without reading. "notes/log.yaml": "note: a deployment log\n", }, @@ -578,7 +595,7 @@ def no_yaml(name, **kwargs): def test_hidden_object_file_in_a_container(self, make_inventory): """A resource id may hold a dot, but a hidden file names no entity.""" - files = {**MINIMAL, "resources/.cable.yaml": "type: Cable\n"} + files = {**MINIMAL, "resources/.cable.yaml": "object_type: Cable\n"} with pytest.raises(InvalidInventoryError, match="hidden"): make_inventory(files) @@ -586,7 +603,7 @@ def test_epoch_finer_than_a_nanosecond(self, make_inventory): """A name stating more precision than is kept would load as another instant.""" files = { "acquisitions/DAS.L001..RAW@2024-05-12T103000.1234567899.yaml": ( - "type: Acquisition\n" + "object_type: Acquisition\n" ) } with pytest.raises(InvalidInventoryError, match="finer than the nanosecond"): @@ -597,10 +614,11 @@ def test_unquoted_dates(self, make_inventory): out = make_inventory( { "fiber_arrays/DAS.L001.yaml": ( - "type: FiberArray\nstart_time: 2024-01-01\nend_time: 2024-07-01\n" + "object_type: FiberArray\nstart_time: 2024-01-01\n" + "end_time: 2024-07-01\n" ), "acquisitions/DAS.L001..RAW@2024-02-01.yaml": ( - "type: Acquisition\nend_time: 2024-03-01\n" + "object_type: Acquisition\nend_time: 2024-03-01\n" ), } ) @@ -610,8 +628,8 @@ def test_unquoted_dates(self, make_inventory): def test_type_which_is_not_a_name(self, make_inventory): """A type which is not a name at all names no model either.""" - files = {"acquisitions/DAS.L001..RAW.yaml": "type: [Acquisition]\n"} - with pytest.raises(InvalidInventoryError, match="declares no type"): + files = {"acquisitions/DAS.L001..RAW.yaml": "object_type: [Acquisition]\n"} + with pytest.raises(InvalidInventoryError, match="declares no object_type"): make_inventory(files) @@ -621,8 +639,8 @@ class TestOneIdentityOneSpelling: def test_two_extensions(self, make_inventory): """The same name with two extensions is one identity spelled twice.""" files = { - "resources/cable_01.yaml": "type: Cable\n", - "resources/cable_01.json": '{"type": "Cable"}', + "resources/cable_01.yaml": "object_type: Cable\n", + "resources/cable_01.json": '{"object_type": "Cable"}', } with pytest.raises(InvalidInventoryError, match="two extensions"): make_inventory(files) @@ -636,8 +654,8 @@ def test_two_extensions(self, make_inventory): def test_case_only_difference(self, make_inventory): """A case-insensitive filesystem could not hold both.""" files = { - "fiber_arrays/DAS.L001.yaml": "type: FiberArray\n", - "fiber_arrays/das.l001.yaml": "type: FiberArray\n", + "fiber_arrays/DAS.L001.yaml": "object_type: FiberArray\n", + "fiber_arrays/das.l001.yaml": "object_type: FiberArray\n", } with pytest.raises(InvalidInventoryError, match="differ only by case"): make_inventory(files) @@ -655,8 +673,8 @@ def test_case_only_difference_is_named_as_such(self, tmp_path): def test_file_and_directory(self, make_inventory): """Both spellings of one identity at once raise.""" files = { - "fiber_arrays/DAS.L001.yaml": "type: FiberArray\n", - "fiber_arrays/DAS.L001/attrs.yaml": "type: FiberArray\n", + "fiber_arrays/DAS.L001.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", } with pytest.raises(InvalidInventoryError, match="file and a directory"): make_inventory(files) @@ -665,8 +683,8 @@ def test_envelope_spelled_twice(self, make_inventory): """The envelope is one file however it is spelled.""" files = { **MINIMAL, - "inventory.yaml": "type: Inventory\n", - "inventory.json": '{"type": "Inventory"}', + "inventory.yaml": "object_type: Inventory\n", + "inventory.json": '{"object_type": "Inventory"}', } with pytest.raises(InvalidInventoryError, match="more than once"): make_inventory(files) @@ -674,8 +692,10 @@ def test_envelope_spelled_twice(self, make_inventory): def test_two_names_for_one_epoch(self, make_inventory): """Epoch-name uniqueness is temporal rather than textual.""" files = { - "acquisitions/DAS.L001..RAW@2024-06-01.yaml": "type: Acquisition\n", - "acquisitions/DAS.L001..RAW@2024-06-01T000000.yaml": "type: Acquisition\n", + "acquisitions/DAS.L001..RAW@2024-06-01.yaml": "object_type: Acquisition\n", + "acquisitions/DAS.L001..RAW@2024-06-01T000000.yaml": ( + "object_type: Acquisition\n" + ), } with pytest.raises(InvalidInventoryError, match="overlap in time"): make_inventory(files) @@ -684,9 +704,9 @@ def test_overlapping_epochs_of_one_entity(self, make_inventory): """Two epochs of one entity may not overlap, not merely coincide.""" files = { "acquisitions/DAS.L001..RAW@2024-01-01.yaml": ( - "type: Acquisition\nend_time: '2024-08-01'\n" + "object_type: Acquisition\nend_time: '2024-08-01'\n" ), - "acquisitions/DAS.L001..RAW@2024-06-01.yaml": "type: Acquisition\n", + "acquisitions/DAS.L001..RAW@2024-06-01.yaml": "object_type: Acquisition\n", } # Both files are named, since the entity alone would not say which. with pytest.raises(InvalidInventoryError, match="RAW@2024-01-01"): @@ -704,7 +724,7 @@ def test_missing_attrs_file(self, make_inventory): def test_stray_object_file(self, make_inventory): """An entity directory's object file is named attrs.""" - files = {"fiber_arrays/DAS.L001/array.yaml": "type: FiberArray\n"} + files = {"fiber_arrays/DAS.L001/array.yaml": "object_type: FiberArray\n"} with pytest.raises(InvalidInventoryError, match="holds only its attrs"): make_inventory(files) @@ -714,7 +734,7 @@ def test_field_note_beside_an_attrs_file(self, make_inventory): { **MINIMAL, "fiber_arrays/DAS.L001/attrs.yaml": ( - "type: FiberArray\nname: from a directory\n" + "object_type: FiberArray\nname: from a directory\n" ), # Declares nothing, so it participates in nothing -- exactly # as it would one directory deeper. @@ -726,8 +746,8 @@ def test_field_note_beside_an_attrs_file(self, make_inventory): def test_attrs_spelled_twice(self, make_inventory): """One identity is spelled once inside the directory too.""" files = { - "fiber_arrays/DAS.L001/attrs.yaml": "type: FiberArray\n", - "fiber_arrays/DAS.L001/attrs.json": '{"type": "FiberArray"}', + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/attrs.json": '{"object_type": "FiberArray"}', } with pytest.raises(InvalidInventoryError, match="more than once"): make_inventory(files) @@ -751,7 +771,7 @@ def test_hidden_and_foreign_entries_in_an_entity(self, tmp_path): { **MINIMAL, "fiber_arrays/DAS.L001/attrs.yaml": ( - "type: FiberArray\nname: from a directory\n" + "object_type: FiberArray\nname: from a directory\n" ), "fiber_arrays/DAS.L001/photos/wellhead.jpg": "not text", "fiber_arrays/DAS.L001/notes.txt": "a note\n", @@ -776,7 +796,7 @@ class TestSeams: def test_track_table_in_an_entity_directory(self, make_inventory): """A track table is refused by name rather than ignored.""" files = { - "fiber_arrays/DAS.L001/attrs.yaml": "type: FiberArray\n", + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", "fiber_arrays/DAS.L001/coupling.csv": "start_distance\n0\n", } with pytest.raises(InvalidInventoryError, match="track table"): @@ -785,9 +805,9 @@ def test_track_table_in_an_entity_directory(self, make_inventory): def test_optical_path_epoch_directory(self, make_inventory): """An optical path epoch is refused by name rather than ignored.""" files = { - "fiber_arrays/DAS.L001/attrs.yaml": "type: FiberArray\n", + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", "fiber_arrays/DAS.L001/path@2024-05-12T103000/attrs.yaml": ( - "type: OpticalPath\n" + "object_type: OpticalPath\n" ), } with pytest.raises(InvalidInventoryError, match="optical path epoch"): @@ -808,7 +828,9 @@ class TestEpochTimestamps: ) def test_malformed(self, make_inventory, stamp): """A name which claims an epoch and gets it wrong raises.""" - files = {f"acquisitions/DAS.L001..RAW@{stamp}.yaml": "type: Acquisition\n"} + files = { + f"acquisitions/DAS.L001..RAW@{stamp}.yaml": "object_type: Acquisition\n" + } with pytest.raises(InvalidInventoryError, match="epoch timestamp"): make_inventory(files) @@ -817,7 +839,9 @@ def test_malformed(self, make_inventory, stamp): ) def test_timezone_designator(self, make_inventory, stamp): """Naive means UTC, so a designator is refused rather than ignored.""" - files = {f"acquisitions/DAS.L001..RAW@{stamp}.yaml": "type: Acquisition\n"} + files = { + f"acquisitions/DAS.L001..RAW@{stamp}.yaml": "object_type: Acquisition\n" + } with pytest.raises(InvalidInventoryError, match="timezone designator"): make_inventory(files) @@ -825,7 +849,7 @@ def test_negative_offset(self, make_inventory): """An offset in the time portion is a designator, not a malformed time.""" files = { "acquisitions/DAS.L001..RAW@2024-05-12T103000-0600.yaml": ( - "type: Acquisition\n" + "object_type: Acquisition\n" ) } with pytest.raises(InvalidInventoryError, match="timezone designator"): @@ -838,7 +862,9 @@ def test_negative_offset(self, make_inventory): ) def test_outside_the_representable_range(self, make_inventory, stamp): """A nanosecond timestamp wraps silently, so the name is refused.""" - files = {f"acquisitions/DAS.L001..RAW@{stamp}.yaml": "type: Acquisition\n"} + files = { + f"acquisitions/DAS.L001..RAW@{stamp}.yaml": "object_type: Acquisition\n" + } with pytest.raises(InvalidInventoryError, match="outside the range"): make_inventory(files) @@ -847,7 +873,7 @@ def test_the_last_representable_instant(self, make_inventory): out = make_inventory( { "acquisitions/DAS.L001..RAW@2262-04-11T234716.yaml": ( - "type: Acquisition\n" + "object_type: Acquisition\n" ) } ) @@ -858,7 +884,7 @@ def test_two_epoch_markers(self, make_inventory): """A name carries at most one epoch.""" files = { "acquisitions/DAS.L001..RAW@2024-06-01@2024-07-01.yaml": ( - "type: Acquisition\n" + "object_type: Acquisition\n" ) } with pytest.raises(InvalidInventoryError, match="more than one"): @@ -866,13 +892,16 @@ def test_two_epoch_markers(self, make_inventory): def test_empty_epoch(self, make_inventory): """A trailing marker names no epoch.""" - files = {"acquisitions/DAS.L001..RAW@.yaml": "type: Acquisition\n"} + files = {"acquisitions/DAS.L001..RAW@.yaml": "object_type: Acquisition\n"} with pytest.raises(InvalidInventoryError, match="names no epoch"): make_inventory(files) def test_resources_have_no_epochs(self, make_inventory): """A resource is not time-ranged, so its name states no epoch.""" - files = {**MINIMAL, "resources/int_01@2024-06-01.yaml": "type: Interrogator\n"} + files = { + **MINIMAL, + "resources/int_01@2024-06-01.yaml": "object_type: Interrogator\n", + } with pytest.raises(InvalidInventoryError, match="have none"): make_inventory(files) @@ -884,9 +913,9 @@ def test_child_outside_every_epoch(self, make_inventory): """A child falling in no epoch of its container is misfiled.""" files = { "fiber_arrays/DAS.L001.yaml": ( - "type: FiberArray\nend_time: '2024-06-01'\n" + "object_type: FiberArray\nend_time: '2024-06-01'\n" ), - "acquisitions/DAS.L001..RAW@2024-07-01.yaml": "type: Acquisition\n", + "acquisitions/DAS.L001..RAW@2024-07-01.yaml": "object_type: Acquisition\n", } with pytest.raises(InvalidInventoryError, match="0 epochs effective"): make_inventory(files) @@ -895,10 +924,10 @@ def test_ambiguous_child(self, make_inventory): """An unset start beside several container epochs is ambiguous.""" files = { "fiber_arrays/DAS.L001.yaml": ( - "type: FiberArray\nend_time: '2024-06-01'\n" + "object_type: FiberArray\nend_time: '2024-06-01'\n" ), - "fiber_arrays/DAS.L001@2024-06-01.yaml": "type: FiberArray\n", - "acquisitions/DAS.L001..RAW.yaml": "type: Acquisition\n", + "fiber_arrays/DAS.L001@2024-06-01.yaml": "object_type: FiberArray\n", + "acquisitions/DAS.L001..RAW.yaml": "object_type: Acquisition\n", } with pytest.raises(InvalidInventoryError, match="2 epochs effective at any"): make_inventory(files) @@ -907,9 +936,9 @@ def test_station_placed_in_a_network_epoch(self, make_inventory): """Networks epoch like everything else which is time-ranged.""" out = make_inventory( { - "networks/DAS.yaml": "type: Network\nend_time: '2024-06-01'\n", - "networks/DAS@2024-06-01.yaml": "type: Network\nname: later\n", - "stations/DAS.STA1@2024-07-01.yaml": "type: Station\n", + "networks/DAS.yaml": "object_type: Network\nend_time: '2024-06-01'\n", + "networks/DAS@2024-06-01.yaml": "object_type: Network\nname: later\n", + "stations/DAS.STA1@2024-07-01.yaml": "object_type: Station\n", } ) by_name = {x.name: x for x in out.networks} @@ -922,20 +951,23 @@ class TestEnvelope: def test_wrong_type(self, make_inventory): """The envelope declares its type under the same rule as any file.""" - files = {**MINIMAL, "inventory.yaml": "type: Network\n"} + files = {**MINIMAL, "inventory.yaml": "object_type: Network\n"} with pytest.raises(InvalidInventoryError, match="envelope declares"): make_inventory(files) @pytest.mark.parametrize("field", ["networks", "resources"]) def test_collections_are_refused(self, make_inventory, field): """The collections live in the directory structure.""" - files = {**MINIMAL, "inventory.yaml": f"type: Inventory\n{field}: []\n"} + files = {**MINIMAL, "inventory.yaml": f"object_type: Inventory\n{field}: []\n"} with pytest.raises(InvalidInventoryError, match="directory structure"): make_inventory(files) def test_unknown_field_names_the_file(self, make_inventory): """A typo in the envelope says which file states it.""" - files = {**MINIMAL, "inventory.yaml": "type: Inventory\nschema_verison: 1\n"} + files = { + **MINIMAL, + "inventory.yaml": "object_type: Inventory\nschema_verison: 1\n", + } with pytest.raises(InvalidInventoryError, match=r"inventory\.yaml"): make_inventory(files) @@ -971,12 +1003,31 @@ def test_empty_container_is_an_empty_inventory(self, tmp_path): class TestModelAssumptions: """Pin what this loader assumes about the models it builds.""" - def test_only_union_members_carry_a_type_field(self): - """The type tag is a real field only where it discriminates a union.""" + def test_only_union_members_declare_the_tag_as_a_field(self): + """The tag is a real field only where it discriminates a union.""" checked = [] for name, container in loader._CONTAINERS.items(): for model in container.models: - assert ("type" in model.model_fields) == (name == "resources") + assert (TAG_FIELD in model.model_fields) == (name == "resources") + checked.append(model) + assert len(checked) == 9 + + def test_every_model_reads_its_own_tag(self): + """ + The loader leaves the tag in the data for the model to check. + + Before every model read its own, the loader had to pop the tag for + the ones which do not declare it, since they forbid extra fields. + """ + checked = [] + for container in loader._CONTAINERS.values(): + for model in container.models: + # The addressed models are named by their file, so a code + # is the one thing they will not default. + fields = {"code": "X"} if "code" in model.model_fields else {} + assert isinstance(model(**fields, **{TAG_FIELD: model.__name__}), model) + with pytest.raises(ValidationError): + model(**fields, **{TAG_FIELD: "Inventory"}) checked.append(model) assert len(checked) == 9 @@ -992,5 +1043,5 @@ def test_epoch_bearing_models_are_time_ranged(self): assert len(checked) == len(loader._CONTAINERS) == 5 def test_inventory_models_forbid_extra_fields(self): - """Type must be popped: an inventory model refuses unknown input.""" + """An inventory model refuses unknown input, tag included.""" assert InventoryModel.model_config["extra"] == "forbid" diff --git a/tests/test_models.py b/tests/test_models.py index 402fcfde0..3ec69084d 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -2,20 +2,27 @@ from __future__ import annotations +import json import pickle +import subprocess +import sys import numpy as np import pytest from pydantic import Field, ValidationError from dascore.core.attrs import PatchAttrs +from dascore.core.inventory import Cable +from dascore.exceptions import InvalidModelTagError from dascore.io.core import FiberIO +from dascore.io.sintela.core import SintelaPatchAttrs from dascore.models import ( DascoreBaseModel, DateTime64, FrozenDictType, TimeDelta64, UnitQuantity, + registry, sensible_model_equals, ) from dascore.units import Quantity @@ -27,6 +34,18 @@ class _TestModel(DascoreBaseModel): some_str: str = "10" +class _Inner(DascoreBaseModel): + """A nested model, which a document must also name.""" + + value: int = 1 + + +class _Outer(DascoreBaseModel): + """A model holding another.""" + + inner: _Inner = Field(default_factory=_Inner) + + class TestModelEquals: """Tests for seeing if models/dicts are equal.""" @@ -227,3 +246,179 @@ def test_no_field_defaults_to_a_non_finite_number(self, attrs_class): assert np.isfinite(default), ( f"{attrs_class.__name__}.{name} defaults to {default}." ) + + +@pytest.fixture +def clean_registry(): + """Undo whatever a test registers, so the real registry is untouched.""" + before = registry.registered_models() + yield + registry._REGISTRY.clear() + registry._REGISTRY.update(before) + + +def _model_in(module: str, name: str = "Square", base=DascoreBaseModel): + """Declare a model as though it lived in another package.""" + return type(name, (base,), {"__module__": module, "__qualname__": name}) + + +class TestModelTagRegistry: + """A tag names one class, and the registry is what resolves it.""" + + def test_dascore_models_register_bare(self): + """A bare name means dascore, which keeps files hand-authorable.""" + assert registry.registered_models()["PatchAttrs"] is PatchAttrs + assert registry.get_model_tag(PatchAttrs) == "PatchAttrs" + + def test_out_of_tree_models_are_namespaced(self, clean_registry): + """A plugin's namespace is derived, so it needs no ceremony.""" + cls = _model_in("myplugin.shapes") + assert registry.get_model_tag(cls) == "myplugin:Square" + assert registry.registered_models()["myplugin:Square"] is cls + + def test_colliding_dascore_names_raise(self, clean_registry): + """Two of our own classes may not claim one tag; this is the pin.""" + first = _model_in("dascore.somewhere", "Doubled") + with pytest.raises(InvalidModelTagError, match="claim the tag"): + _model_in("dascore.elsewhere", "Doubled") + # The collision does not replace what was there before it. + assert registry.registered_models()["Doubled"] is first + + def test_colliding_plugin_names_warn(self, clean_registry): + """A user cannot rename another package's class, so this only warns.""" + _model_in("myplugin.a", "Doubled") + with pytest.warns(UserWarning, match="claim the tag"): + second = _model_in("myplugin.b", "Doubled") + assert registry.registered_models()["myplugin:Doubled"] is second + + def test_a_class_declared_in_a_function_is_not_registered(self, clean_registry): + """Nothing can resolve a name which exists only while a call runs.""" + + class Local(DascoreBaseModel): + """A model which cannot be addressed from a document.""" + + assert "Local" not in registry.registered_models() + + def test_a_reimported_module_replaces_its_own_entry(self, clean_registry): + """Re-importing a module is not two classes claiming one tag.""" + _model_in("myplugin.shapes") + second = _model_in("myplugin.shapes") + assert registry.registered_models()["myplugin:Square"] is second + + @pytest.mark.parametrize( + "tag", ["Cable", "myplugin:Square", "my_plugin.sub:Square", "Cable-0.0.1"] + ) + def test_legal_tags(self, tag): + """The grammar takes a name, a namespace and room for a version.""" + assert registry.TAG_PATTERN.match(tag) + + @pytest.mark.parametrize( + "tag", ["", "9Cable", "Cable-1", ":Cable", "dascore.core.inventory.Cable", 3] + ) + def test_illegal_tags_are_refused(self, tag): + """A tag which is not a tag is a malformed document, not an unknown one.""" + with pytest.raises(InvalidModelTagError, match="is not a legal"): + registry.resolve_model_tag(tag) + + def test_an_unknown_tag_falls_back_with_a_warning(self): + """A document from an uninstalled package still reads as its base.""" + data = {registry.TAG_FIELD: "absent:Whatever"} + with pytest.warns(UserWarning, match="Nothing registers"): + out = registry.resolve_tagged_model(data, default=PatchAttrs) + assert out is PatchAttrs + + def test_an_unknown_tag_without_a_default_raises(self): + """A standalone document has no other class to fall back on.""" + data = {registry.TAG_FIELD: "absent:Whatever"} + with pytest.raises(InvalidModelTagError, match="Nothing registers"): + registry.resolve_tagged_model(data) + + def test_an_untagged_document_without_a_default_raises(self): + """Nothing but the document says what a standalone document holds.""" + with pytest.raises(InvalidModelTagError, match="declares no"): + registry.resolve_tagged_model({"a": 1}) + + def test_an_untagged_document_takes_the_default(self): + """A caller which names the class does not need the document to.""" + assert registry.resolve_tagged_model({}, default=PatchAttrs) is PatchAttrs + + def test_a_model_in_an_unimported_module_is_found(self): + """ + A format's models only exist once its module is imported. + + Run in a fresh interpreter because the io modules are imported by + the time any other test runs, which is exactly what hides this. + """ + code = ( + "import dascore.models.registry as r\n" + "assert 'ODH4PatchAttrs' not in r.registered_models(), 'already there'\n" + "assert r.resolve_model_tag('ODH4PatchAttrs') is not None, 'not found'\n" + ) + out = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True + ) + assert out.returncode == 0, out.stderr + + +class TestObjectTypeSerialization: + """Every model names its class in a text document, and nowhere else.""" + + def test_json_states_the_class(self): + """A document says what it holds.""" + dumped = json.loads(PatchAttrs(tag="a").model_dump_json()) + assert dumped[registry.TAG_FIELD] == "PatchAttrs" + + def test_a_python_dump_is_untagged(self): + """ + The tag is not a field, and python dumps are not documents. + + Equality compares them, `new` reconstructs from them and the spool + index ingests them; a key which is not a field belongs in none. + """ + assert registry.TAG_FIELD not in PatchAttrs().model_dump() + assert registry.TAG_FIELD not in PatchAttrs().new(tag="a").model_dump() + + def test_equality_is_unaffected(self, clean_registry): + """Two classes' instances are still told apart by their fields.""" + assert PatchAttrs(tag="a") == PatchAttrs(tag="a") + assert PatchAttrs(tag="a") != PatchAttrs(tag="b") + + def test_nested_models_state_their_class(self): + """Universal, so a nested object can be read on its own later.""" + dumped = json.loads(_Outer(inner=_Inner()).model_dump_json()) + assert dumped[registry.TAG_FIELD] == "tests:_Outer" + assert dumped["inner"][registry.TAG_FIELD] == "tests:_Inner" + + def test_a_subclass_reads_back_through_its_base(self): + """The document holds everything the base declares, so this is fine.""" + attrs = SintelaPatchAttrs(gauge_length=10.0) + out = PatchAttrs.model_validate_json(attrs.model_dump_json()) + assert out.gauge_length == 10.0 + + def test_a_foreign_tag_is_refused(self): + """A document which names another class is misfiled, not reinterpreted.""" + data = {registry.TAG_FIELD: "Cable"} + with pytest.raises(ValidationError, match="cannot be read as"): + PatchAttrs(**data) + + def test_an_unknown_tag_is_accepted(self): + """The caller named the class, so there is nothing to disagree with.""" + attrs = PatchAttrs(**{registry.TAG_FIELD: "absent:Whatever"}) + assert isinstance(attrs, PatchAttrs) + + def test_the_tag_does_not_become_an_extra_field(self): + """PatchAttrs keeps extras, and the tag is not one of them.""" + attrs = PatchAttrs(**{registry.TAG_FIELD: "PatchAttrs"}) + assert not hasattr(attrs, registry.TAG_FIELD) + + def test_a_union_member_states_it_once(self): + """ + The five resource models declare the tag as a real field. + + Pydantic must pick a class before an object exists, so their tag + cannot be a serializer concern; the base class leaves them alone + rather than writing a second copy of what they already wrote. + """ + text = Cable(resource_id="c1").model_dump_json() + assert json.loads(text)[registry.TAG_FIELD] == "Cable" + assert text.count(f'"{registry.TAG_FIELD}"') == 1 From 4d7621132e7db793d3e6b2a6b67a8ada5956886f Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Thu, 13 Aug 2026 12:39:33 +0200 Subject: [PATCH 5/8] Record the attrs class in DASDAE files DASDAE writes attr values one at a time into HDF5 attrs rather than one document, so nothing carried the class and every custom PatchAttrs came back as the base. The class is now named beside the values, and read through the registry: a file which names none, or names a format which is not installed, still reads as plain attrs. --- dascore/io/dasdae/utils.py | 26 +++++++++++- tests/test_io/test_dasdae/test_dasdae.py | 53 ++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/dascore/io/dasdae/utils.py b/dascore/io/dasdae/utils.py index bf7673ef6..80cf8b2e0 100644 --- a/dascore/io/dasdae/utils.py +++ b/dascore/io/dasdae/utils.py @@ -18,6 +18,7 @@ from dascore.io.core import make_scan_payload from dascore.io.dasdae._compat import strip_legacy_coord_fields, translate_legacy_attrs from dascore.io.utils import get_exact_coord +from dascore.models.registry import TAG_FIELD, get_model_tag, resolve_tagged_model from dascore.utils.array import ( convert_bytes_to_strings, convert_strings_to_bytes, @@ -34,6 +35,10 @@ # Root marker set on files whose patch attr namespace holds only true attrs. # Files without it may mix flat coord metadata into attrs (see _compat). _SEPARATE_ATTRS_KEY = "__attrs_coords_separate__" +# Names the attrs class a patch group holds. A sibling of the attr +# namespace rather than a member of it, since attrs allow extras and a +# patch may carry one spelled like this key. +_ATTRS_CLASS_KEY = "__attrs_class__" # --- Functions for writing DASDAE format @@ -90,6 +95,9 @@ def _save_attrs_and_dims(patch, patch_group): patch_group.attrs[f"{_ATTR_PREFIX}{i}"] = encoded if attr_type is not None: patch_group.attrs[f"{_ATTR_TYPE_PREFIX}{i}"] = attr_type + # Values are dumped one at a time rather than as one document, so the + # class is recorded beside them rather than injected into them. + patch_group.attrs[_ATTRS_CLASS_KEY] = get_model_tag(type(patch.attrs)) patch_group.attrs["_dims"] = ",".join(patch.dims) @@ -170,6 +178,20 @@ def _get_attrs(patch_group): return out +def _get_attrs_class(patch_group) -> type[PatchAttrs]: + """ + Return the attrs class a patch group names, or the base class. + + A file written before the class was recorded names nothing, and one + written by a format which is no longer installed names something + unresolvable; both read as plain attrs, which is what such a file + always used to give. + """ + tag = unbyte(patch_group.attrs.get(_ATTRS_CLASS_KEY, None)) + data = {TAG_FIELD: tag} if tag else {} + return resolve_tagged_model(data, default=PatchAttrs, source=patch_group.name) + + def _read_array(table_array): """Read an array into numpy.""" data = table_array[:] @@ -299,7 +321,7 @@ def _read_patch(patch_group, legacy: bool = True, **kwargs): coords = _get_coords(patch_group, dims, {}) attr_info = attrs attr_info["_source_patch_id"] = patch_group.name.rsplit("/", maxsplit=1)[-1] - attrs = PatchAttrs.from_dict(attr_info) + attrs = _get_attrs_class(patch_group).from_dict(attr_info) # Note, previously this was wrapped with try, except (Index, KeyError) # and the data = np.array(None) in except block. Not sure, why, removed # try except. @@ -361,7 +383,7 @@ def _get_scan_payload_from_group(group, legacy: bool = True, snap=True): dtype = str(data_node.dtype) if data_node is not None else "" shape = tuple(data_node.shape) if data_node is not None else () return make_scan_payload( - attrs=PatchAttrs.from_dict(attr_info), + attrs=_get_attrs_class(group).from_dict(attr_info), coords=coords, dims=dims, shape=shape, diff --git a/tests/test_io/test_dasdae/test_dasdae.py b/tests/test_io/test_dasdae/test_dasdae.py index 533128f98..fd47d041e 100644 --- a/tests/test_io/test_dasdae/test_dasdae.py +++ b/tests/test_io/test_dasdae/test_dasdae.py @@ -21,6 +21,7 @@ from dascore.io.dasdae._compat import translate_legacy_attrs from dascore.io.dasdae.core import DASDAEV1 from dascore.io.dasdae.utils import ( + _ATTRS_CLASS_KEY, _SEPARATE_ATTRS_KEY, _decode_attr_value, _decode_legacy_attr_value, @@ -33,6 +34,7 @@ _save_array, _save_patch, ) +from dascore.io.odh4.core import ODH4PatchAttrs from dascore.utils.downloader import fetch from dascore.utils.misc import register_func from dascore.utils.time import to_datetime64 @@ -337,6 +339,57 @@ def test_get_patch_summary_has_file_metadata(self, random_spool): assert out["source_patch_id"].notnull().all() +class TestAttrsClassRoundTrip: + """A file names the attrs class it holds, so a custom one comes back.""" + + @pytest.fixture(scope="class") + def odh4_patch(self, random_patch): + """A patch carrying a format's own attrs class.""" + attrs = ODH4PatchAttrs(**dict(random_patch.attrs), gauge_length=10.0) + return random_patch.update(attrs=attrs) + + @pytest.fixture(scope="class") + def odh4_path(self, odh4_patch, tmp_path_factory): + """The patch above, written to a DASDAE file.""" + path = tmp_path_factory.mktemp("attrs_class") / "odh4.h5" + odh4_patch.io.write(path, "dasdae") + return path + + def test_read_keeps_the_class(self, odh4_path): + """Reading rebuilds the class which was written.""" + attrs = dc.read(odh4_path)[0].attrs + assert isinstance(attrs, ODH4PatchAttrs) + assert attrs.gauge_length == 10.0 + + def test_scan_keeps_the_class(self, odh4_path): + """Scanning takes the same path, so it says the same thing.""" + assert isinstance(dc.scan(odh4_path)[0].attrs, ODH4PatchAttrs) + + def test_a_file_naming_no_class_reads_as_patch_attrs(self, odh4_path, tmp_path): + """A file written before the class was recorded still reads.""" + path = tmp_path / "unnamed.h5" + shutil.copy(odh4_path, path) + with h5py.File(path, "a") as h5: + for group in h5["waveforms"].values(): + del group.attrs[_ATTRS_CLASS_KEY] + attrs = dc.read(path)[0].attrs + assert type(attrs) is dc.PatchAttrs + # The values were always kept; it is the class which was lost. + assert attrs.gauge_length == 10.0 + + def test_an_unresolvable_class_warns_and_falls_back(self, odh4_path, tmp_path): + """An archive written by a package we lack is still readable.""" + path = tmp_path / "foreign.h5" + shutil.copy(odh4_path, path) + with h5py.File(path, "a") as h5: + for group in h5["waveforms"].values(): + group.attrs[_ATTRS_CLASS_KEY] = "absent:ForeignAttrs" + with pytest.warns(UserWarning, match="Nothing registers"): + attrs = dc.read(path)[0].attrs + assert type(attrs) is dc.PatchAttrs + assert attrs.gauge_length == 10.0 + + class TestLegacyFixtureCompatibility: """Tests for the retained legacy DASDAE fixture compatibility helpers.""" From fded3036a05a9e65cfa2d48831cb0b2d24bbe097 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Thu, 13 Aug 2026 12:47:53 +0200 Subject: [PATCH 6/8] Check a resolved class is the kind the caller asked for Type checking caught the hole: the registry holds every model, so a file naming any of them resolved, and a DASDAE file whose class key said 'Cable' would have been handed to a reader expecting attrs. Document the subclasses and how an optional number is spelled. --- dascore/models/registry.py | 17 ++++++++++--- docs/contributing/new_format.qmd | 5 +++- docs/notes/patch_attrs.qmd | 43 ++++++++++++++++++++++++++++++++ tests/test_models.py | 18 +++++++++++++ 4 files changed, 79 insertions(+), 4 deletions(-) diff --git a/dascore/models/registry.py b/dascore/models/registry.py index b9ff54100..6515ecdcd 100644 --- a/dascore/models/registry.py +++ b/dascore/models/registry.py @@ -12,10 +12,13 @@ import re import warnings from collections.abc import Mapping +from typing import TypeVar, cast from dascore.exceptions import InvalidModelTagError from dascore.utils.plugins import get_entry_point_loaders +T = TypeVar("T") + # The key a document states its class in. Specific enough that no reader's # header and no user's extra attribute is expected to spell it, which is why # the validator may consume it wherever it appears. @@ -159,9 +162,9 @@ def check_tag_matches(cls: type, tag: str) -> None: def resolve_tagged_model( data: Mapping, - default: type | None = None, + default: type[T] | None = None, source: str | None = None, -) -> type: +) -> type[T]: """ Return the model class a document names. @@ -187,7 +190,15 @@ def resolve_tagged_model( raise InvalidModelTagError(msg) return default if (cls := resolve_model_tag(tag)) is not None: - return cls + # The registry holds every model, so what it returns is only the + # right kind of thing because this says so. + if default is not None and not issubclass(cls, default): + msg = ( + f"The document{where} declares {TAG_FIELD} {tag!r}, which is " + f"not a {default.__name__}." + ) + raise InvalidModelTagError(msg) + return cast("type[T]", cls) msg = ( f"Nothing registers the {TAG_FIELD} {tag!r}{where}. It was likely " "written by a package which is not installed." diff --git a/docs/contributing/new_format.qmd b/docs/contributing/new_format.qmd index c27c28f4f..dcc05e18c 100644 --- a/docs/contributing/new_format.qmd +++ b/docs/contributing/new_format.qmd @@ -192,7 +192,10 @@ name and who may change it later. whatever name the format uses. These are welcome, but each new one must be listed in the `VENDOR_ATTRS` set in `tests/test_io/test_common_io.py`, which is where a reviewer checks that the value is not one of the facts - above wearing a vendor's name. + above wearing a vendor's name. Declaring them on a `PatchAttrs` subclass + gets them validated and coerced; see + [format-specific subclasses](`docs/notes/patch_attrs.qmd`) for how one is + spelled, and why an optional number is never defaulted to nan. Stay as close to the file's own spelling as the rules above allow. A value that is one of the facts in (2) has to take the canonical name, because two diff --git a/docs/notes/patch_attrs.qmd b/docs/notes/patch_attrs.qmd index 3977949d9..1bfab8719 100644 --- a/docs/notes/patch_attrs.qmd +++ b/docs/notes/patch_attrs.qmd @@ -39,3 +39,46 @@ DASCore-assigned `data_type` values should be snake_case and listed in `VALID_DA | `data_type="some_value"` | Set the returned patch's `data_type` to that value. | Functions may still require a specific input label with `required_attrs`, for example `required_attrs={"data_type": "velocity"}`. This should only be used when the function's assumptions truly depend on that label and are documented. + +## Format-specific subclasses + +A format whose files carry fields the base class does not model declares a +`PatchAttrs` subclass for them; there are more than a dozen under +`dascore/io/`. Since the fields are declared, they are validated and coerced +like any other, rather than surviving as untyped extras. + +A serialized patch names the class it holds, so the subclass comes back: + +```python +import dascore as dc +from dascore.io.odh4.core import ODH4PatchAttrs + +patch = dc.get_example_patch() +patch = patch.update(attrs=ODH4PatchAttrs(**dict(patch.attrs), gauge_length=10.0)) +patch.io.write("example.h5", "dasdae") + +type(dc.read("example.h5")[0].attrs) # ODH4PatchAttrs +``` + +The name is a registered one rather than an import path, so moving a class +between modules does not break files already written, and reading a file +never imports a path it names. A file naming a class which is not installed +— written, say, by a plugin this environment lacks — reads as the base +`PatchAttrs` with a warning rather than failing. + +### Spelling an optional number + +An absent number is spelled `FiniteFloat | None`, defaulting to `None`: + +```python +from dascore.models import FiniteFloat + +class JingleV1PatchAttrs(dc.PatchAttrs): + """Attrs for the jingle format.""" + + gauge_length: FiniteFloat | None = None +``` + +Not `float = np.nan`. JSON has no spelling for nan, so such a field writes +`null` and then refuses to read it back, which means the class cannot +reconstruct from its own `model_dump_json()`. diff --git a/tests/test_models.py b/tests/test_models.py index 3ec69084d..d476f7c61 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -327,6 +327,12 @@ def test_an_unknown_tag_falls_back_with_a_warning(self): out = registry.resolve_tagged_model(data, default=PatchAttrs) assert out is PatchAttrs + def test_a_resolved_tag_must_be_the_kind_asked_for(self): + """A file naming a class of the wrong kind is refused, not built.""" + data = {registry.TAG_FIELD: "Cable"} + with pytest.raises(InvalidModelTagError, match="not a PatchAttrs"): + registry.resolve_tagged_model(data, default=PatchAttrs) + def test_an_unknown_tag_without_a_default_raises(self): """A standalone document has no other class to fall back on.""" data = {registry.TAG_FIELD: "absent:Whatever"} @@ -342,6 +348,18 @@ def test_an_untagged_document_takes_the_default(self): """A caller which names the class does not need the document to.""" assert registry.resolve_tagged_model({}, default=PatchAttrs) is PatchAttrs + def test_a_broken_plugin_does_not_break_resolution(self, monkeypatch): + """A plugin which cannot be imported has no models to find.""" + + def _boom(): + raise ImportError("this plugin is a stale entry point") + + monkeypatch.setattr(registry, "_plugins_swept", False) + monkeypatch.setattr( + registry, "get_entry_point_loaders", lambda group: {"broken": _boom} + ) + assert registry.resolve_model_tag("absent:Whatever") is None + def test_a_model_in_an_unimported_module_is_found(self): """ A format's models only exist once its module is imported. From 63fbcc9c7651ac1de30e173605b062744bb7e7e4 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Thu, 13 Aug 2026 13:22:12 +0200 Subject: [PATCH 7/8] Address the adversarial review A tag DASCore could write but not read: a package whose name starts with a capital, which is legal and common, produced a namespace the grammar refused, and the refusal came before the fallback so such a file could not be read at all. The grammar takes it, and a class which still cannot be named is not named rather than named unreadably. An absent number is now OptionalFiniteFloat, which reads a non-finite one as absent instead of refusing it. Refusing was the worse half of the nan migration: readers hand vendor header floats straight to these classes, and a scan swallows a ValidationError as 'failed to scan', so one NaN gauge length would have dropped a file out of a spool silently. Four bare float fields the first pass missed are migrated with them. A value which names no known class is left alone rather than consumed, since attrs keep extra fields and eating one loses a reader's metadata. Two classes claiming one tag now stop it resolving instead of quietly picking the last registered. A union member's tag survives exclude flags. PatchSummary.attrs serializes as what it holds, or the tag names a class whose fields the document lacks. The mute geometries keep DASCore's equality, which their ndarray fields need. Tests: the union-member test counted json keys, which a dict cannot duplicate; it now pins the value through a subclass whose tag and Literal disagree. The class walk asserts a floor, since a plugin failing to import would otherwise shrink it silently. The round trip compares text, not models, because equality counts every null equal to every other. --- dascore/core/inventory.py | 9 +- dascore/core/summary.py | 4 +- dascore/io/ai4eps/core.py | 10 +- dascore/io/ap_sensing/core.py | 6 +- dascore/io/core.py | 6 +- dascore/io/dasdae/utils.py | 12 +- dascore/io/febus/core.py | 6 +- dascore/io/gdr/core.py | 3 +- dascore/io/neubrex/core.py | 7 +- dascore/io/odh4/core.py | 6 +- dascore/io/optodas/core.py | 4 +- dascore/io/prodml/utils.py | 12 +- dascore/io/silixah5/core.py | 6 +- dascore/io/sintela/core.py | 4 +- dascore/io/sintela/protobuf_utils.py | 9 +- dascore/io/sr4731/utils.py | 10 +- dascore/io/xml_binary/core.py | 6 +- dascore/models/__init__.py | 2 + dascore/models/base.py | 53 ++++-- dascore/models/registry.py | 174 +++++++++++------- dascore/models/types.py | 25 ++- dascore/proc/mute.py | 13 +- dascore/utils/models.py | 51 +----- dascore/utils/plugins.py | 3 + docs/contributing/new_format.qmd | 2 +- docs/notes/patch_attrs.qmd | 13 +- docs/tutorial/patch.qmd | 2 +- tests/test_core/test_inventory_loader.py | 19 +- tests/test_models.py | 213 +++++++++++++++++++---- 29 files changed, 456 insertions(+), 234 deletions(-) diff --git a/dascore/core/inventory.py b/dascore/core/inventory.py index c5e8e67a5..ef1de8c7a 100644 --- a/dascore/core/inventory.py +++ b/dascore/core/inventory.py @@ -132,11 +132,12 @@ def _object_type_tag(name: str): """ Return the serialization-only ``object_type`` field of a union member. - Every model states its class when serialized (see + Every model states its class in a json document (see [dascore.models.registry](`dascore.models.registry`)), but pydantic must - pick a class for these before an object exists, so the models sharing a - union declare the tag as a real field and the base class leaves them - alone. Users never set it: it defaults to the class name, the Literal + pick a union member's class before an object exists, so these models + declare the tag as a real field and the base class leaves them alone. + + Users never set it: it defaults to the class name, the Literal annotation rejects any other value, and it is hidden from repr. """ return Field(default=name, repr=False) diff --git a/dascore/core/summary.py b/dascore/core/summary.py index 72749c084..a49c62cbe 100644 --- a/dascore/core/summary.py +++ b/dascore/core/summary.py @@ -11,7 +11,7 @@ import numpy as np import pandas as pd -from pydantic import ConfigDict, Field, model_validator +from pydantic import ConfigDict, Field, SerializeAsAny, model_validator import dascore as dc from dascore.constants import path_types @@ -202,7 +202,7 @@ class PatchSummary(DascoreBaseModel): model_config = ConfigDict(title="Patch Summary", extra="ignore", frozen=True) - attrs: PatchAttrs = Field(default_factory=PatchAttrs) + attrs: SerializeAsAny[PatchAttrs] = Field(default_factory=PatchAttrs) coords: dict[str, CoordSummary] = Field(default_factory=dict) dims: tuple[str, ...] = () shape: tuple[int, ...] = () diff --git a/dascore/io/ai4eps/core.py b/dascore/io/ai4eps/core.py index 659e3efb3..0192fecc8 100644 --- a/dascore/io/ai4eps/core.py +++ b/dascore/io/ai4eps/core.py @@ -9,7 +9,7 @@ import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload -from dascore.models import DateTime64, FiniteFloat +from dascore.models import DateTime64, OptionalFiniteFloat from dascore.utils.hdf5 import H5Reader from .utils import _get_attrs_dict, _get_coords, _get_patches, _is_ai4eps @@ -20,11 +20,11 @@ class AI4EPSPatchAttrs(dc.PatchAttrs): event_id: str = "" event_time: DateTime64 = np.datetime64("NaT", "ns") - magnitude: FiniteFloat | None = None + magnitude: OptionalFiniteFloat = None magnitude_type: str = "" - event_latitude: FiniteFloat | None = None - event_longitude: FiniteFloat | None = None - event_depth_km: FiniteFloat | None = None + event_latitude: OptionalFiniteFloat = None + event_longitude: OptionalFiniteFloat = None + event_depth_km: OptionalFiniteFloat = None class AI4EPSV1(FiberIO): diff --git a/dascore/io/ap_sensing/core.py b/dascore/io/ap_sensing/core.py index d2d176a10..e33c832fd 100644 --- a/dascore/io/ap_sensing/core.py +++ b/dascore/io/ap_sensing/core.py @@ -9,7 +9,7 @@ import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload -from dascore.models import FiniteFloat +from dascore.models import OptionalFiniteFloat from dascore.utils.hdf5 import H5Reader from .utils import _get_attrs_dict, _get_coords, _get_patches, _get_version_string @@ -18,8 +18,8 @@ class APSensingPatchAttrs(dc.PatchAttrs): """Patch Attributes for AP sensing.""" - gauge_length: FiniteFloat | None = None - radians_to_nano_strain: FiniteFloat | None = None + gauge_length: OptionalFiniteFloat = None + radians_to_nano_strain: OptionalFiniteFloat = None class APSensingV10(FiberIO): diff --git a/dascore/io/core.py b/dascore/io/core.py index 71d530e64..c53cf9075 100644 --- a/dascore/io/core.py +++ b/dascore/io/core.py @@ -77,7 +77,7 @@ warn_or_raise, ) from dascore.utils.paths import coerce_to_local_path, coerce_to_upath, is_local_path -from dascore.utils.plugins import get_entry_point_loaders +from dascore.utils.plugins import FIBER_IO_GROUP, get_entry_point_loaders from dascore.utils.progress import track from dascore.utils.remote_io import ( get_remote_cache_scope, @@ -490,7 +490,7 @@ def _eps(self): Get the unloaded entry points registered to this domain into a dict of {name: ep}. """ - return pd.Series(get_entry_point_loaders("dascore.fiber_io")) + return pd.Series(get_entry_point_loaders(FIBER_IO_GROUP)) @cached_property @_locked("_lock") @@ -940,7 +940,7 @@ class FiberIO: # True when a single resource can hold more than one patch. multi_patch_write: bool = False - manager = _FiberIOManager("dascore.fiber_io") + manager = _FiberIOManager(FIBER_IO_GROUP) # A dict of methods which should implement automatic type casting. # and the index of the parameter to type cast. diff --git a/dascore/io/dasdae/utils.py b/dascore/io/dasdae/utils.py index 80cf8b2e0..c6aea96b1 100644 --- a/dascore/io/dasdae/utils.py +++ b/dascore/io/dasdae/utils.py @@ -18,7 +18,7 @@ from dascore.io.core import make_scan_payload from dascore.io.dasdae._compat import strip_legacy_coord_fields, translate_legacy_attrs from dascore.io.utils import get_exact_coord -from dascore.models.registry import TAG_FIELD, get_model_tag, resolve_tagged_model +from dascore.models.registry import get_model_tag, resolve_tagged_model from dascore.utils.array import ( convert_bytes_to_strings, convert_strings_to_bytes, @@ -96,8 +96,11 @@ def _save_attrs_and_dims(patch, patch_group): if attr_type is not None: patch_group.attrs[f"{_ATTR_TYPE_PREFIX}{i}"] = attr_type # Values are dumped one at a time rather than as one document, so the - # class is recorded beside them rather than injected into them. - patch_group.attrs[_ATTRS_CLASS_KEY] = get_model_tag(type(patch.attrs)) + # class is recorded beside them rather than injected into them. A class + # which cannot be named (see get_model_tag) is simply not named, which + # reads back the way a file written before this did. + if (tag := get_model_tag(type(patch.attrs))) is not None: + patch_group.attrs[_ATTRS_CLASS_KEY] = tag patch_group.attrs["_dims"] = ",".join(patch.dims) @@ -188,8 +191,7 @@ def _get_attrs_class(patch_group) -> type[PatchAttrs]: always used to give. """ tag = unbyte(patch_group.attrs.get(_ATTRS_CLASS_KEY, None)) - data = {TAG_FIELD: tag} if tag else {} - return resolve_tagged_model(data, default=PatchAttrs, source=patch_group.name) + return resolve_tagged_model(tag or None, default=PatchAttrs) def _read_array(table_array): diff --git a/dascore/io/febus/core.py b/dascore/io/febus/core.py index 08319dcc7..590dde056 100644 --- a/dascore/io/febus/core.py +++ b/dascore/io/febus/core.py @@ -16,7 +16,7 @@ ) from dascore.io import FiberIO, ScanPayload from dascore.io.core import make_scan_payload -from dascore.models import FiniteFloat, UTF8Str +from dascore.models import OptionalFiniteFloat, UTF8Str from dascore.utils.hdf5 import H5Reader from dascore.utils.io import TextReader @@ -59,8 +59,8 @@ class FebusPatchAttrs(dc.PatchAttrs): The zone designations """ - gauge_length: FiniteFloat | None = None - pulse_length: FiniteFloat | None = None + gauge_length: OptionalFiniteFloat = None + pulse_length: OptionalFiniteFloat = None group: str = "" source: str = "" diff --git a/dascore/io/gdr/core.py b/dascore/io/gdr/core.py index a55e34a76..b1bfd2842 100644 --- a/dascore/io/gdr/core.py +++ b/dascore/io/gdr/core.py @@ -14,13 +14,14 @@ from dascore.io import FiberIO, ScanPayload, make_scan_payload from dascore.io.gdr.utils_das import _get_attrs_coords_and_data, _get_version from dascore.io.utils import build_patches +from dascore.models import OptionalFiniteFloat from dascore.utils.hdf5 import H5Reader class GDRPatchAttrs(dc.PatchAttrs): """Patch attrs for GDR files.""" - gauge_length: float + gauge_length: OptionalFiniteFloat project_number: str = "" diff --git a/dascore/io/neubrex/core.py b/dascore/io/neubrex/core.py index 4aa9bcb89..5a6307cd8 100644 --- a/dascore/io/neubrex/core.py +++ b/dascore/io/neubrex/core.py @@ -13,6 +13,7 @@ import dascore.io.neubrex.utils_rfs as rfs_utils from dascore.io import FiberIO, ScanPayload, make_scan_payload from dascore.io.utils import build_patches +from dascore.models import OptionalFiniteFloat from dascore.utils.hdf5 import H5Reader @@ -29,10 +30,10 @@ class NeubrexRFSPatchAttrs(dc.PatchAttrs): class NeubrexDASPatchAttrs(dc.PatchAttrs): """Patch attrs for Neubrex DAS Format files.""" - gauge_length: float = 0 - index_of_reflection: float = 1.46 + gauge_length: OptionalFiniteFloat = 0 + index_of_reflection: OptionalFiniteFloat = 1.46 triggered_time: np.datetime64 | None = None - phase_to_strain: float | None = None + phase_to_strain: OptionalFiniteFloat = None distance_decimation_filter: int = 0 time_decimation_filter: int = 0 diff --git a/dascore/io/odh4/core.py b/dascore/io/odh4/core.py index 4b01eb841..781abb1ef 100644 --- a/dascore/io/odh4/core.py +++ b/dascore/io/odh4/core.py @@ -7,7 +7,7 @@ import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload -from dascore.models import FiniteFloat +from dascore.models import OptionalFiniteFloat from dascore.utils.hdf5 import H5Reader from .utils import _get_attrs_dict, _get_coords, _get_patches, _is_odh4, _read_attrs @@ -16,8 +16,8 @@ class ODH4PatchAttrs(dc.PatchAttrs): """Patch attributes for ODH4 files.""" - gauge_length: FiniteFloat | None = None - scale_factor_to_strain: FiniteFloat | None = None + gauge_length: OptionalFiniteFloat = None + scale_factor_to_strain: OptionalFiniteFloat = None class ODH4V1(FiberIO): diff --git a/dascore/io/optodas/core.py b/dascore/io/optodas/core.py index c756b3755..1ce7fc0a5 100644 --- a/dascore/io/optodas/core.py +++ b/dascore/io/optodas/core.py @@ -7,7 +7,7 @@ import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload -from dascore.models import FiniteFloat, UTF8Str +from dascore.models import OptionalFiniteFloat, UTF8Str from dascore.utils.hdf5 import H5Reader from .utils import _get_opto_das_attrs, _get_opto_das_version_str, _read_opto_das @@ -16,7 +16,7 @@ class OptoDASPatchAttrs(dc.PatchAttrs): """Patch attrs for OptoDAS.""" - gauge_length: FiniteFloat | None = None + gauge_length: OptionalFiniteFloat = None schema_version: UTF8Str = "" diff --git a/dascore/io/prodml/utils.py b/dascore/io/prodml/utils.py index 1251a5ff7..1b12b7e27 100644 --- a/dascore/io/prodml/utils.py +++ b/dascore/io/prodml/utils.py @@ -15,7 +15,7 @@ from dascore.core.coords import get_coord from dascore.exceptions import InvalidSpoolError, PatchError from dascore.io.utils import convert_attr_units, get_exact_coord -from dascore.models import FiniteFloat, UTF8Str +from dascore.models import OptionalFiniteFloat, UTF8Str from dascore.units import get_quantity_str from dascore.utils.hdf5 import encode_h5_strings from dascore.utils.io import _normalize_source_patch_ids @@ -77,8 +77,8 @@ class ProdMLRawPatchAttrs(dc.PatchAttrs): """Patch attrs for raw data contained in ProdML.""" - pulse_width: FiniteFloat | None = None - gauge_length: FiniteFloat | None = None + pulse_width: OptionalFiniteFloat = None + gauge_length: OptionalFiniteFloat = None schema_version: UTF8Str = "" @@ -86,13 +86,13 @@ class ProdMLFbePatchAttrs(ProdMLRawPatchAttrs): """Patch attrs for fbe (frequency band extracted) data in Prodml.""" raw_reference: UTF8Str = "" - transform_size: FiniteFloat | None = None + transform_size: OptionalFiniteFloat = None transform_type: UTF8Str = "" window_size: int | None = None window_function: UTF8Str = "" window_overlap: int | None = None - start_frequency: float = 0 - end_frequency: FiniteFloat | None = None + start_frequency: OptionalFiniteFloat = 0 + end_frequency: OptionalFiniteFloat = None @dataclass diff --git a/dascore/io/silixah5/core.py b/dascore/io/silixah5/core.py index c177b0d01..2efcdad06 100644 --- a/dascore/io/silixah5/core.py +++ b/dascore/io/silixah5/core.py @@ -9,7 +9,7 @@ import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload -from dascore.models import FiniteFloat +from dascore.models import OptionalFiniteFloat from dascore.utils.hdf5 import H5Reader from .utils import ( @@ -25,8 +25,8 @@ class SilixaPatchAttrs(dc.PatchAttrs): """Patch Attributes for Silixa hdf5 format.""" - gauge_length: FiniteFloat | None = None - pulse_width: FiniteFloat | None = None + gauge_length: OptionalFiniteFloat = None + pulse_width: OptionalFiniteFloat = None class SilixaH5V1(FiberIO): diff --git a/dascore/io/sintela/core.py b/dascore/io/sintela/core.py index e4245723c..ad3edc032 100644 --- a/dascore/io/sintela/core.py +++ b/dascore/io/sintela/core.py @@ -11,7 +11,7 @@ import dascore as dc from dascore.constants import opt_timeable_types from dascore.io import FiberIO, ScanPayload, make_scan_payload -from dascore.models import FiniteFloat +from dascore.models import OptionalFiniteFloat from dascore.utils.io import BinaryReader, LocalBinaryReader from .protobuf_utils import get_supported_family_tag, read_payload, scan_payload @@ -27,7 +27,7 @@ class SintelaPatchAttrs(dc.PatchAttrs): """Patch Attributes for Sintela binary format.""" - gauge_length: FiniteFloat | None = None + gauge_length: OptionalFiniteFloat = None class SintelaBinaryV3(FiberIO): diff --git a/dascore/io/sintela/protobuf_utils.py b/dascore/io/sintela/protobuf_utils.py index 34e574034..5bc31f027 100644 --- a/dascore/io/sintela/protobuf_utils.py +++ b/dascore/io/sintela/protobuf_utils.py @@ -56,7 +56,7 @@ from dascore.core.coords import get_coord from dascore.exceptions import InvalidFiberFileError from dascore.io.core import ScanPayload, make_scan_payload -from dascore.models import FiniteFloat, PositiveFiniteFloat, PositiveInt +from dascore.models import OptionalFiniteFloat, PositiveFiniteFloat, PositiveInt from dascore.utils.misc import optional_import, suppress_warnings PBUF_MAGIC = 0x46554250 @@ -143,7 +143,7 @@ def _get_fft_data_type(has_complex: bool) -> dict[str, str]: class SintelaProtobufAttrs(PatchAttrs): """Patch attributes for Sintela protobuf recordings.""" - gauge_length: FiniteFloat | None = None + gauge_length: OptionalFiniteFloat = None packet_type: str = "" recorder_namespace: str = "" metadata_recording_time: np.datetime64 | None = None @@ -158,8 +158,9 @@ class _ProtobufModel(BaseModel): Base for this module's parsing models. Plain pydantic: these validate values on the way out of a protobuf - payload and are never serialized, so they want none of what - DascoreBaseModel adds beyond validation. + payload and are never serialized. Subclassing DascoreBaseModel would + only claim each a tag in the model registry, naming them in documents + they never appear in. """ model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) diff --git a/dascore/io/sr4731/utils.py b/dascore/io/sr4731/utils.py index ca1ac60fc..6d793a251 100644 --- a/dascore/io/sr4731/utils.py +++ b/dascore/io/sr4731/utils.py @@ -35,7 +35,7 @@ from dascore.exceptions import InvalidFiberFileError from dascore.io.core import ScanPayload, make_scan_payload from dascore.io.utils import build_patches -from dascore.models import FiniteFloat +from dascore.models import OptionalFiniteFloat DIMS = ("time", "distance") REQUIRED_BLOCKS = frozenset( @@ -57,10 +57,10 @@ class Block: class SR4731PatchAttrs(PatchAttrs): """Patch attributes for supported SR-4731 SOR files.""" - wavelength_nm: FiniteFloat | None = None - acquisition_range_m: FiniteFloat | None = None - sample_spacing_usec: FiniteFloat | None = None - refractive_index: FiniteFloat | None = None + wavelength_nm: OptionalFiniteFloat = None + acquisition_range_m: OptionalFiniteFloat = None + sample_spacing_usec: OptionalFiniteFloat = None + refractive_index: OptionalFiniteFloat = None trace_count: int = 0 sample_scale: int = 0 diff --git a/dascore/io/xml_binary/core.py b/dascore/io/xml_binary/core.py index f64b7f856..301ee2ede 100644 --- a/dascore/io/xml_binary/core.py +++ b/dascore/io/xml_binary/core.py @@ -9,7 +9,7 @@ import dascore as dc from dascore.io import FiberIO, ScanPayload -from dascore.models import FiniteFloat, UTF8Str +from dascore.models import OptionalFiniteFloat, UTF8Str from dascore.utils.paths import coerce_to_upath from .utils import _load_patches, _paths_to_scan_patches, _read_xml_metadata @@ -18,8 +18,8 @@ class BinaryPatchAttrs(dc.PatchAttrs): """Patch attrs for Binary.""" - pulse_width: FiniteFloat | None = None - gauge_length: FiniteFloat | None = None + pulse_width: OptionalFiniteFloat = None + gauge_length: OptionalFiniteFloat = None zone_name: UTF8Str = "" diff --git a/dascore/models/__init__.py b/dascore/models/__init__.py index 798c90677..977bc613a 100644 --- a/dascore/models/__init__.py +++ b/dascore/models/__init__.py @@ -23,6 +23,7 @@ DTypeLike, FiniteFloat, FrozenDictType, + OptionalFiniteFloat, PositiveFiniteFloat, PositiveInt, TimeDelta64, @@ -41,6 +42,7 @@ "FiniteFloat", "FrozenDictType", "InventoryModel", + "OptionalFiniteFloat", "PositiveFiniteFloat", "PositiveInt", "TimeDelta64", diff --git a/dascore/models/base.py b/dascore/models/base.py index 4f3e00bf7..972ddcec1 100644 --- a/dascore/models/base.py +++ b/dascore/models/base.py @@ -23,6 +23,7 @@ from dascore.exceptions import InvalidInventoryError from dascore.models.registry import ( TAG_FIELD, + _lookup, check_tag_matches, get_model_tag, register_model, @@ -120,18 +121,27 @@ def _write_object_type( self, handler: SerializerFunctionWrapHandler, info: SerializationInfo ) -> Any: """ - Name this class in the document, in text serializations only. + Name this class in the document, in json-mode dumps only. - Only json mode, because a python-mode dump is not a document: it is - what equality compares, what ``new`` reconstructs from and what the - index ingests, none of which want a key that is not a field. + A python-mode dump is not a document: it is what equality compares, + what ``new`` reconstructs from and what the index ingests, none of + which want a key that is not a field. - Models which discriminate a union declare ``object_type`` as a real - field, and pydantic has already written it. + Gated inside the serializer rather than with ``when_used="json"``, + which reads better and silently breaks ``include``/``exclude``: + pydantic skips the whole wrapper in python mode, and with it the + field filtering the handler would have applied. """ out = handler(self) - if info.mode == "json" and TAG_FIELD not in type(self).model_fields: - out[TAG_FIELD] = get_model_tag(type(self)) + if info.mode != "json" or TAG_FIELD in out: # a union member's own + return out + # Its field is defaulted, so exclude_defaults drops it and leaves a + # document the union cannot dispatch on. Put back what the model + # says rather than what the registry calls the class: a subclass + # out of tree has a different tag, and the Literal would refuse it. + declared = getattr(self, TAG_FIELD, None) + if (tag := declared or get_model_tag(type(self))) is not None: + out[TAG_FIELD] = tag return out @model_validator(mode="before") @@ -140,17 +150,28 @@ def _read_object_type(cls, data: Any) -> Any: """ Consume a document's class tag, refusing one which names another class. - The tag is never required: a document dispatches on it before it gets - here, and a hand-written object or a nested one may simply not state - it. What it may not do is disagree. + The tag is never required: a document dispatches on it before it + gets here, and a hand-written object or a nested one may simply not + state it. What it may not do is disagree. + + A value which names no known class is left alone rather than + consumed. ``PatchAttrs`` keeps extra fields, so this key may be a + reader's own metadata, and eating it would lose that silently; a + tag DASCore wrote always resolves. """ - if not isinstance(data, Mapping) or TAG_FIELD in cls.model_fields: + if not isinstance(data, Mapping) or TAG_FIELD not in data: + return data + # A union member declares the tag as a real field and validates it + # against its own Literal, which is stricter than this. + if TAG_FIELD in cls.model_fields: return data - if TAG_FIELD not in data: + tag = data[TAG_FIELD] + if (declared := _lookup(tag)) is None: return data - data = dict(data) - check_tag_matches(cls, data.pop(TAG_FIELD)) - return data + check_tag_matches(cls, declared, tag) + out = dict(data) + out.pop(TAG_FIELD) + return out def new(self, **kwargs) -> Self: """Create new instance with some attributed updated.""" diff --git a/dascore/models/registry.py b/dascore/models/registry.py index 6515ecdcd..7ed7d5129 100644 --- a/dascore/models/registry.py +++ b/dascore/models/registry.py @@ -10,18 +10,16 @@ from __future__ import annotations import re +import threading import warnings -from collections.abc import Mapping from typing import TypeVar, cast from dascore.exceptions import InvalidModelTagError -from dascore.utils.plugins import get_entry_point_loaders +from dascore.utils.plugins import FIBER_IO_GROUP, get_entry_point_loaders T = TypeVar("T") -# The key a document states its class in. Specific enough that no reader's -# header and no user's extra attribute is expected to spell it, which is why -# the validator may consume it wherever it appears. +# The key a document states its class in. TAG_FIELD = "object_type" NAMESPACE_SEP = ":" @@ -30,16 +28,22 @@ # `object_type: Cable` and a plugin's says `object_type: myplugin:Square`. DASCORE_NAMESPACE = "dascore" -FIBER_IO_GROUP = "dascore.fiber_io" - # `[namespace:]ClassName[-x.y.z]`. A python class name holds neither ":" -# nor "-", so the three parts can never be read for one another. Nothing -# reads a version today; the grammar only keeps room for one, where an -# absent version will mean the earliest. -TAG_PATTERN = re.compile(r"^(?:[a-z_][\w.]*:)?[A-Za-z_]\w*(?:-\d+\.\d+\.\d+)?$") +# nor "-", so the three parts can never be read for one another. The +# namespace accepts a capital because package names do (Terra15, PIL). +# Nothing reads a version today; the grammar only keeps room for one, +# where an absent version will mean the earliest. +TAG_PATTERN = re.compile(r"^(?:[A-Za-z_][\w.]*:)?[A-Za-z_]\w*(?:-\d+\.\d+\.\d+)?$") _REGISTRY: dict[str, type] = {} +# Tags two different classes have claimed. Kept rather than resolved to +# either: a file written by the first would otherwise be read as the +# second, silently applying the wrong model to it. +_AMBIGUOUS: dict[str, tuple[str, str]] = {} + +_sweep_lock = threading.Lock() + # Set once the io plugins have been swept looking for an unresolved tag. _plugins_swept = False @@ -51,17 +55,22 @@ def _derive_namespace(cls: type) -> str: return cls.__module__.split(".", 1)[0] -def get_model_tag(cls: type) -> str: +def get_model_tag(cls: type) -> str | None: """ - Return the tag which names a model class in a document. + Return the tag which names a model class in a document, if it has one. An out-of-tree class is namespaced by the package which declares it; - DASCore's own classes are bare. + DASCore's own classes are bare. None means the class cannot be named: + a parametrized generic is spelled ``G[int]``, which no document could + state and this module could not read back. Writing such a tag anyway + would produce a file DASCore refuses to read. """ namespace = _derive_namespace(cls) if namespace == DASCORE_NAMESPACE: - return cls.__name__ - return f"{namespace}{NAMESPACE_SEP}{cls.__name__}" + tag = cls.__name__ + else: + tag = f"{namespace}{NAMESPACE_SEP}{cls.__name__}" + return tag if TAG_PATTERN.match(tag) else None def register_model(cls: type) -> None: @@ -70,14 +79,15 @@ def register_model(cls: type) -> None: Classes declared inside a function are skipped: nothing can resolve a name which only exists while its enclosing call runs, and two of them - sharing a name is neither a mistake nor resolvable. + sharing a name is neither a mistake nor resolvable. So are classes + whose derived tag is not a legal one; see `get_model_tag`. """ - if "" in cls.__qualname__: + if "" in cls.__qualname__ or (tag := get_model_tag(cls)) is None: return - tag = get_model_tag(cls) existing = _REGISTRY.get(tag) if existing is not None and _identity(existing) != _identity(cls): _report_collision(tag, existing, cls) + return # A module re-imported under the same name replaces its own entry. _REGISTRY[tag] = cls @@ -87,45 +97,78 @@ def _identity(cls: type) -> tuple[str, str]: return (cls.__module__, cls.__qualname__) +def _spell(cls: type) -> str: + """Spell a class the way a collision message needs to.""" + return f"{cls.__module__}.{cls.__qualname__}" + + def _report_collision(tag: str, existing: type, new: type) -> None: """Complain that two different classes want one tag.""" msg = ( - f"Two models claim the tag {tag!r}: {existing.__module__}." - f"{existing.__qualname__} and {new.__module__}.{new.__qualname__}. " - "A tag must name one class; rename one of them." + f"Two models claim the tag {tag!r}: {_spell(existing)} and " + f"{_spell(new)}. A tag must name one class; rename one of them." ) # DASCore's own names are its own to keep unique, and a test pins it. - # Out of tree the collision may be between two packages a user merely - # installed, which they cannot fix by renaming, so it warns and the - # last registration wins -- as duplicate entry points already do. if _derive_namespace(new) == DASCORE_NAMESPACE: raise InvalidModelTagError(msg) - warnings.warn(msg, UserWarning, stacklevel=2) + # Out of tree the collision may be between two packages a user merely + # installed, which they cannot fix by renaming, so importing them both + # still works. What the tag may not do is quietly resolve to one of + # them: a file written by the first would then be read as the second. + _AMBIGUOUS[tag] = (_spell(existing), _spell(new)) + _REGISTRY.pop(tag, None) + warnings.warn(f"{msg} Documents naming it can no longer be read.", UserWarning) def _sweep_plugin_modules() -> None: - """Import the io plugins, defining any models they declare.""" + """ + Import the io plugins, defining any models they declare. + + Not `FiberIO.manager.load_plugins()`, which does the same thing and + more: `dascore.io.core` imports `dascore.core.attrs`, which imports + this module, so naming it here at module scope is a cycle -- and a + function-level import is banned by PLC0415. + """ global _plugins_swept if _plugins_swept: return - _plugins_swept = True - for loader in get_entry_point_loaders(FIBER_IO_GROUP).values(): - try: - loader() - except Exception: - # A plugin which cannot be imported has no models to find. It is - # not reported here: FiberIO warns about the same plugin when it - # loads formats, and a failure to resolve one tag is not the - # place to announce an unrelated broken install. - continue + with _sweep_lock: + # The flag is set after the imports rather than before, so a thread + # arriving mid-sweep waits here rather than reading a registry which + # is still filling and concluding a class is not installed. Sweeping + # again after that wait costs nothing: every module it names is by + # then in sys.modules. + for loader in get_entry_point_loaders(FIBER_IO_GROUP).values(): + try: + loader() + except Exception: + # A plugin which cannot be imported has no models to find. + # It is not reported here: FiberIO warns about the same + # plugin when it loads formats, and failing to resolve one + # tag is not the place to announce a broken install. + continue + _plugins_swept = True + + +def _lookup(tag: object) -> type | None: + """Return the class a tag names, or None if nothing usable names it.""" + if not isinstance(tag, str) or not TAG_PATTERN.match(tag) or tag in _AMBIGUOUS: + return None + if (cls := _REGISTRY.get(tag)) is not None: + return cls + # A format's models only exist once its module is imported, and io + # modules are imported lazily, so an unknown name is worth one sweep. + _sweep_plugin_modules() + return _REGISTRY.get(tag) def resolve_model_tag(tag: str) -> type | None: """ Return the class a tag names, or None if nothing registers it. - Raises if the tag is not a legal tag at all, which is a malformed - document rather than an unknown class. + Raises if the tag could never name a class -- it is not a legal tag, + or two classes have claimed it -- which is a document this process + cannot read rather than one whose class it merely does not have. """ if not isinstance(tag, str) or not TAG_PATTERN.match(tag): msg = ( @@ -133,25 +176,25 @@ def resolve_model_tag(tag: str) -> type | None: "optionally namespaced, eg 'Cable' or 'myplugin:Square'." ) raise InvalidModelTagError(msg) - if (cls := _REGISTRY.get(tag)) is not None: - return cls - # A format's models only exist once its module is imported, and io - # modules are imported lazily, so an unknown name is worth one sweep. - _sweep_plugin_modules() - return _REGISTRY.get(tag) + if tag in _AMBIGUOUS: + first, second = _AMBIGUOUS[tag] + msg = ( + f"The {TAG_FIELD} {tag!r} names two classes, {first} and " + f"{second}, so which one wrote a document cannot be known." + ) + raise InvalidModelTagError(msg) + return _lookup(tag) -def check_tag_matches(cls: type, tag: str) -> None: +def check_tag_matches(cls: type, declared: type, tag: str) -> None: """ - Refuse a document whose tag names a class the one being built is not. + Refuse a document whose tag names some other class. A tag naming a subclass is accepted: such a document holds everything the class being built declares, which is what a caller asking for the - base class asked for. An unregistered tag is accepted too -- the caller - named the class, so there is nothing for the document to disagree with. + base class asked for. """ - declared = resolve_model_tag(tag) - if declared is None or issubclass(declared, cls): + if issubclass(declared, cls): return msg = ( f"A document declaring {TAG_FIELD} {tag!r} cannot be read as " @@ -161,25 +204,26 @@ def check_tag_matches(cls: type, tag: str) -> None: def resolve_tagged_model( - data: Mapping, + tag: str | None, default: type[T] | None = None, source: str | None = None, ) -> type[T]: """ - Return the model class a document names. + Return the model class a tag names. Parameters ---------- - data - The document, which states its class in its ``object_type`` key. + tag + What the document declared, or None if it declared nothing. default - The class to fall back on when the document names one which is not + The class to fall back on when the tag names one which is not registered, usually because a plugin which wrote it is not - installed. Without one, an unresolved name raises. + installed. Without one, an unresolved tag raises. source - Where the document came from, used in messages. + Where the document came from, named in errors. Deliberately not in + the warning below, which would otherwise defeat the de-duplication + that keeps a spool of many files from warning once per file. """ - tag = data.get(TAG_FIELD) if isinstance(data, Mapping) else None where = f" in {source}" if source else "" if tag is None: if default is None: @@ -190,8 +234,8 @@ def resolve_tagged_model( raise InvalidModelTagError(msg) return default if (cls := resolve_model_tag(tag)) is not None: - # The registry holds every model, so what it returns is only the - # right kind of thing because this says so. + # The registry holds every model, not just the caller's kind, so + # this check is what makes the returned class a `default` at all. if default is not None and not issubclass(cls, default): msg = ( f"The document{where} declares {TAG_FIELD} {tag!r}, which is " @@ -200,12 +244,12 @@ def resolve_tagged_model( raise InvalidModelTagError(msg) return cast("type[T]", cls) msg = ( - f"Nothing registers the {TAG_FIELD} {tag!r}{where}. It was likely " - "written by a package which is not installed." + f"Nothing registers the {TAG_FIELD} {tag!r}. It was likely written " + "by a package which is not installed." ) if default is None: - raise InvalidModelTagError(msg) - warnings.warn(f"{msg} Reading it as {default.__name__}.", UserWarning) + raise InvalidModelTagError(f"{msg[:-1]}{where}.") + warnings.warn(f"{msg} Reading it as {default.__name__}.", UserWarning, stacklevel=2) return default diff --git a/dascore/models/types.py b/dascore/models/types.py index 4dd8085d2..833598814 100644 --- a/dascore/models/types.py +++ b/dascore/models/types.py @@ -8,6 +8,7 @@ import numpy as np from pydantic import ( AfterValidator, + BeforeValidator, Field, PlainSerializer, PlainValidator, @@ -96,9 +97,29 @@ def _to_unit_quantity(value): PositiveInt = Annotated[int, Field(gt=0)] # A float which must be finite; nan/inf silently poison downstream math. -# Spell an optional number `FiniteFloat | None`: nan has no JSON spelling, -# so a nan-defaulted float writes `null` and then refuses to read it back. FiniteFloat = Annotated[float, Field(allow_inf_nan=False)] + +def _none_if_not_finite(value): + """Read a non-finite number as an absent one.""" + if value is None: + return None + try: + return value if np.isfinite(value) else None + except (TypeError, ValueError): + # Not a number at all. Whatever it is, the field's own validation + # is what should describe it. + return value + + +# How to spell a number which may be absent. Absence is `None`, because +# json has no spelling for nan: a nan-defaulted float writes `null` and +# then refuses to read it back, so its class cannot reconstruct from its +# own dump. A file which spells "unknown" as nan is read as absent rather +# than refused, since that is the same statement in another notation. +OptionalFiniteFloat = Annotated[ + FiniteFloat | None, BeforeValidator(_none_if_not_finite) +] + # A positive (> 0), finite (no nan/inf) float. PositiveFiniteFloat = Annotated[float, Field(gt=0, allow_inf_nan=False)] diff --git a/dascore/proc/mute.py b/dascore/proc/mute.py index 5b9d2a17c..9d9d4d9c8 100644 --- a/dascore/proc/mute.py +++ b/dascore/proc/mute.py @@ -15,6 +15,7 @@ import dascore as dc from dascore.constants import PatchType from dascore.exceptions import ParameterError +from dascore.models import sensible_model_equals, sensible_model_hash from dascore.utils.docs import compose_docstring from dascore.utils.misc import ( get_2d_line_intersection, @@ -44,13 +45,19 @@ class _MuteGeometry(ABC, BaseModel): """ Parent class for Mute Geometry. - A plain pydantic model: these carry a mute's geometry from the argument - parsing to the mask and are never serialized, so they want validation - and nothing else DascoreBaseModel offers. + A plain pydantic model: these carry a mute's geometry from argument + parsing to mask construction and are never serialized. Subclassing + DascoreBaseModel would only claim each a tag in the model registry, + naming them in documents they never appear in. """ model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + # These hold ndarrays, which pydantic's generated __eq__ cannot compare + # (it raises on the truth value of an array), so they keep DASCore's. + __eq__ = sensible_model_equals + __hash__ = sensible_model_hash + dims: tuple[str, ...] axes: tuple[int, ...] relative: bool = True diff --git a/dascore/utils/models.py b/dascore/utils/models.py index 3a85217e5..aa4ddfca5 100644 --- a/dascore/utils/models.py +++ b/dascore/utils/models.py @@ -2,56 +2,15 @@ Deprecated home of DASCore's models; import from [dascore.models](`dascore.models`). Everything here is re-exported from its new home so out-of-tree readers which -import from this path keep working. +import from this path keep working. Re-exported wholesale rather than listed, +since a second hand-maintained list is a second thing to forget. """ from __future__ import annotations from pydantic import BaseModel -from dascore.models.base import ( - DascoreBaseModel, - InventoryModel, - TimeRangedModel, - sensible_model_equals, - sensible_model_hash, - values_equal, -) -from dascore.models.types import ( - ArrayLike, - CommaSeparatedStr, - DateTime64, - DTypeLike, - FiniteFloat, - FrozenDictType, - PositiveFiniteFloat, - PositiveInt, - TimeDelta64, - UnitQuantity, - UTF8Str, - frozen_dict_serializer, - frozen_dict_validator, -) +from dascore.models import * # noqa: F403 +from dascore.models import __all__ as _models_all -__all__ = [ - "ArrayLike", - "BaseModel", - "CommaSeparatedStr", - "DTypeLike", - "DascoreBaseModel", - "DateTime64", - "FiniteFloat", - "FrozenDictType", - "InventoryModel", - "PositiveFiniteFloat", - "PositiveInt", - "TimeDelta64", - "TimeRangedModel", - "UTF8Str", - "UnitQuantity", - "frozen_dict_serializer", - "frozen_dict_validator", - "sensible_model_equals", - "sensible_model_hash", - "values_equal", -] +__all__ = [*_models_all, "BaseModel"] diff --git a/dascore/utils/plugins.py b/dascore/utils/plugins.py index 11cf1e007..6a9437232 100644 --- a/dascore/utils/plugins.py +++ b/dascore/utils/plugins.py @@ -7,6 +7,9 @@ from importlib.metadata import entry_points from typing import Any +# The entry-point group every FiberIO plugin registers under. +FIBER_IO_GROUP = "dascore.fiber_io" + @functools.cache def get_entry_point_loaders(entry_point_group: str) -> dict[str, Any]: diff --git a/docs/contributing/new_format.qmd b/docs/contributing/new_format.qmd index dcc05e18c..c95a7a490 100644 --- a/docs/contributing/new_format.qmd +++ b/docs/contributing/new_format.qmd @@ -195,7 +195,7 @@ name and who may change it later. above wearing a vendor's name. Declaring them on a `PatchAttrs` subclass gets them validated and coerced; see [format-specific subclasses](`docs/notes/patch_attrs.qmd`) for how one is - spelled, and why an optional number is never defaulted to nan. + spelled, and why an optional number is never defaulted to nan or inf. Stay as close to the file's own spelling as the rules above allow. A value that is one of the facts in (2) has to take the canonical name, because two diff --git a/docs/notes/patch_attrs.qmd b/docs/notes/patch_attrs.qmd index 1bfab8719..483de5b9b 100644 --- a/docs/notes/patch_attrs.qmd +++ b/docs/notes/patch_attrs.qmd @@ -68,17 +68,18 @@ never imports a path it names. A file naming a class which is not installed ### Spelling an optional number -An absent number is spelled `FiniteFloat | None`, defaulting to `None`: +An absent number is spelled `OptionalFiniteFloat`, defaulting to `None`: ```python -from dascore.models import FiniteFloat +from dascore.models import OptionalFiniteFloat class JingleV1PatchAttrs(dc.PatchAttrs): """Attrs for the jingle format.""" - gauge_length: FiniteFloat | None = None + gauge_length: OptionalFiniteFloat = None ``` -Not `float = np.nan`. JSON has no spelling for nan, so such a field writes -`null` and then refuses to read it back, which means the class cannot -reconstruct from its own `model_dump_json()`. +Not `float = np.nan`, and not `float = np.inf`. JSON spells neither, so such +a field writes `null` and then refuses to read it back, which means the class +cannot reconstruct from its own `model_dump_json()`. A non-finite value read +from a file is taken as an absent one rather than refused. diff --git a/docs/tutorial/patch.qmd b/docs/tutorial/patch.qmd index 1415a9a03..9e7b9b33c 100644 --- a/docs/tutorial/patch.qmd +++ b/docs/tutorial/patch.qmd @@ -218,7 +218,7 @@ you intend to select literally. ## Attrs -The metadata stored in `Patch.attrs` is a [pydantic model](https://docs.pydantic.dev/usage/models/) which enforces a schema and provides validation. [`PatchAttrs.get_summary_df`](`dascore.utils.models.DascoreBaseModel.get_summary_df`) generates a table of the attribute descriptions: +The metadata stored in `Patch.attrs` is a [pydantic model](https://docs.pydantic.dev/usage/models/) which enforces a schema and provides validation. [`PatchAttrs.get_summary_df`](`dascore.models.base.DascoreBaseModel.get_summary_df`) generates a table of the attribute descriptions: ```{python} diff --git a/tests/test_core/test_inventory_loader.py b/tests/test_core/test_inventory_loader.py index c159d21fc..3f0feb56f 100644 --- a/tests/test_core/test_inventory_loader.py +++ b/tests/test_core/test_inventory_loader.py @@ -1042,6 +1042,21 @@ def test_epoch_bearing_models_are_time_ranged(self): checked.append(name) assert len(checked) == len(loader._CONTAINERS) == 5 - def test_inventory_models_forbid_extra_fields(self): - """An inventory model refuses unknown input, tag included.""" + def test_inventory_models_refuse_unknown_input(self): + """ + An inventory model refuses a key it does not declare. + + Which is why the tag has to be consumed rather than ignored, and is + checked on the concrete models rather than on the base, since a + subclass may override the config (PatchAttrs does, in the other + hierarchy). + """ assert InventoryModel.model_config["extra"] == "forbid" + checked = [] + for container in loader._CONTAINERS.values(): + for model in container.models: + fields = {"code": "X"} if "code" in model.model_fields else {} + with pytest.raises(ValidationError, match="not permitted"): + model(**fields, nonsense_key=1) + checked.append(model) + assert len(checked) == 9 diff --git a/tests/test_models.py b/tests/test_models.py index d476f7c61..41bd0d78a 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -12,7 +12,7 @@ from pydantic import Field, ValidationError from dascore.core.attrs import PatchAttrs -from dascore.core.inventory import Cable +from dascore.core.inventory import Cable, Inventory from dascore.exceptions import InvalidModelTagError from dascore.io.core import FiberIO from dascore.io.sintela.core import SintelaPatchAttrs @@ -20,6 +20,7 @@ DascoreBaseModel, DateTime64, FrozenDictType, + OptionalFiniteFloat, TimeDelta64, UnitQuantity, registry, @@ -184,6 +185,13 @@ def test_unhashable_field_still_refuses(self): hash(_TestModel(array=np.arange(10))) +# A floor, not a count: raise it when a format adds a class. It exists +# because FiberIO swallows a plugin's import failure as a warning, which +# would otherwise drop that format's class out of the walk below and leave +# this file green while testing one format fewer. +_ATTRS_CLASS_FLOOR = 18 + + def _dascore_patch_attrs_classes(): """Every PatchAttrs class DASCore itself declares, base included.""" # The subclasses only exist once their io modules are imported, and other @@ -197,6 +205,10 @@ def _dascore_patch_attrs_classes(): stack.extend(cls.__subclasses__()) if cls.__module__.startswith("dascore."): found[cls.__name__] = cls + assert len(found) >= _ATTRS_CLASS_FLOOR, ( + f"only {len(found)} PatchAttrs classes were found; a format's plugin " + "probably failed to import, which would silently go untested here." + ) return [found[name] for name in sorted(found)] @@ -205,6 +217,12 @@ def _dascore_patch_attrs_classes(): _REQUIRED_ATTR_VALUES = {"gauge_length": 10.0} +def _optional_float_names(cls): + """Names of the fields on a class which hold an optional number.""" + annotation = OptionalFiniteFloat.__origin__ + return [n for n, f in cls.model_fields.items() if f.annotation == annotation] + + def _minimal_attrs(cls): """Build the emptiest legal instance of a PatchAttrs class.""" required = {name for name, f in cls.model_fields.items() if f.is_required()} @@ -232,21 +250,39 @@ def test_json_round_trip(self, attrs_class): """A defaulted instance reconstructs from its own json.""" attrs = _minimal_attrs(attrs_class) out = attrs_class.model_validate_json(attrs.model_dump_json()) - assert out == attrs + # Compared as text, not with ==, which counts any null equal to any + # other and so cannot see a value degrade to None on the way through. + assert out.model_dump_json() == attrs.model_dump_json() - def test_no_field_defaults_to_a_non_finite_number(self, attrs_class): + def test_no_field_holds_a_number_json_cannot_spell(self, attrs_class): """ - Nan and inf have no json spelling, so a float defaulting to one - writes null and then refuses to read it back. Optional numbers are - spelled `FiniteFloat | None`. + No field holds a number json cannot spell. + + NaN and inf write as `null` and then refuse to read back, so an + optional number is spelled `OptionalFiniteFloat`. Checked on the + built instance rather than the raw default, since `np.float32("nan")` + is not a `float` and would slip past that. """ - for name, field in attrs_class.model_fields.items(): - default = field.get_default(call_default_factory=True) - if isinstance(default, float): - assert np.isfinite(default), ( - f"{attrs_class.__name__}.{name} defaults to {default}." + instance = _minimal_attrs(attrs_class) + for name in attrs_class.model_fields: + value = getattr(instance, name, None) + if isinstance(value, float | np.floating): + assert np.isfinite(value), ( + f"{attrs_class.__name__}.{name} holds {value}." ) + def test_a_non_finite_value_from_a_file_reads_as_absent(self, attrs_class): + """ + A format which spells "unknown" as nan is read, not refused. + + Readers hand vendor header floats straight to these classes, and a + scan swallows a ValidationError as "failed to scan", so refusing one + would silently drop the file from a spool rather than report it. + """ + for name in _optional_float_names(attrs_class): + built = _minimal_attrs(attrs_class).new(**{name: np.float32("nan")}) + assert getattr(built, name) is None + @pytest.fixture def clean_registry(): @@ -262,6 +298,46 @@ def _model_in(module: str, name: str = "Square", base=DascoreBaseModel): return type(name, (base,), {"__module__": module, "__qualname__": name}) +class TestOptionalFiniteFloat: + """How a number which may be absent is spelled.""" + + class _Model(DascoreBaseModel): + value: OptionalFiniteFloat = None + + @pytest.mark.parametrize("value", [np.nan, np.inf, -np.inf, np.float32("nan")]) + def test_a_non_finite_number_is_absent(self, value): + """A file which spells "unknown" as nan is read, not refused.""" + assert self._Model(value=value).value is None + + def test_a_number_survives(self): + """Everything else is the number it was.""" + assert self._Model(value=1.5).value == 1.5 + + def test_a_non_number_is_left_to_the_field(self): + """ + Only numbers can be finite, so anything else passes through here. + + A string is what a yaml or json document holds, and pydantic's own + coercion is what should read it -- or refuse it, in its own words. + """ + assert self._Model(value="1.5").value == 1.5 + with pytest.raises(ValidationError): + self._Model(value="not a number") + + +def test_optional_numbers_are_declared_across_the_formats(): + """ + The nan-to-None loop above tests nothing on a class with no such field. + + So the fields it walks are counted here: were the annotation renamed or + the migration reverted, every one of those loops would quietly empty. + """ + total = sum( + len(_optional_float_names(cls)) for cls in _dascore_patch_attrs_classes() + ) + assert total >= 25 + + class TestModelTagRegistry: """A tag names one class, and the registry is what resolves it.""" @@ -284,12 +360,42 @@ def test_colliding_dascore_names_raise(self, clean_registry): # The collision does not replace what was there before it. assert registry.registered_models()["Doubled"] is first - def test_colliding_plugin_names_warn(self, clean_registry): - """A user cannot rename another package's class, so this only warns.""" + def test_colliding_plugin_names_stop_resolving(self, clean_registry): + """ + A user cannot rename another package's class, so importing both works. + + What the tag may not do is quietly pick one: a document written by + the first would then be read as the second. + """ _model_in("myplugin.a", "Doubled") with pytest.warns(UserWarning, match="claim the tag"): - second = _model_in("myplugin.b", "Doubled") - assert registry.registered_models()["myplugin:Doubled"] is second + _model_in("myplugin.b", "Doubled") + assert "myplugin:Doubled" not in registry.registered_models() + with pytest.raises(InvalidModelTagError, match="names two classes"): + registry.resolve_model_tag("myplugin:Doubled") + + def test_an_uppercase_package_is_a_legal_namespace(self, clean_registry): + """ + A package name may start with a capital, and many do (PIL, Terra15). + + The tag DASCore writes must be one it can read: a namespace the + grammar refused would make its own files unreadable. + """ + cls = _model_in("Terra15.attrs", "Terra15Attrs") + tag = registry.get_model_tag(cls) + assert tag == "Terra15:Terra15Attrs" + assert registry.resolve_model_tag(tag) is cls + + def test_a_class_which_cannot_be_named_is_not_registered(self, clean_registry): + """ + A parametrized generic is spelled `G[int]`, which no document states. + + Writing that tag anyway would produce a file DASCore refuses to + read, so such a class is simply not named. + """ + cls = _model_in("myplugin.generics", "Square[int]") + assert registry.get_model_tag(cls) is None + assert "myplugin:Square[int]" not in registry.registered_models() def test_a_class_declared_in_a_function_is_not_registered(self, clean_registry): """Nothing can resolve a name which exists only while a call runs.""" @@ -310,7 +416,9 @@ def test_a_reimported_module_replaces_its_own_entry(self, clean_registry): ) def test_legal_tags(self, tag): """The grammar takes a name, a namespace and room for a version.""" - assert registry.TAG_PATTERN.match(tag) + # Through the function rather than the pattern: what matters is that + # a legal tag is looked up rather than refused outright. + registry.resolve_model_tag(tag) @pytest.mark.parametrize( "tag", ["", "9Cable", "Cable-1", ":Cable", "dascore.core.inventory.Cable", 3] @@ -322,31 +430,28 @@ def test_illegal_tags_are_refused(self, tag): def test_an_unknown_tag_falls_back_with_a_warning(self): """A document from an uninstalled package still reads as its base.""" - data = {registry.TAG_FIELD: "absent:Whatever"} with pytest.warns(UserWarning, match="Nothing registers"): - out = registry.resolve_tagged_model(data, default=PatchAttrs) + out = registry.resolve_tagged_model("absent:Whatever", default=PatchAttrs) assert out is PatchAttrs def test_a_resolved_tag_must_be_the_kind_asked_for(self): """A file naming a class of the wrong kind is refused, not built.""" - data = {registry.TAG_FIELD: "Cable"} with pytest.raises(InvalidModelTagError, match="not a PatchAttrs"): - registry.resolve_tagged_model(data, default=PatchAttrs) + registry.resolve_tagged_model("Cable", default=PatchAttrs) def test_an_unknown_tag_without_a_default_raises(self): """A standalone document has no other class to fall back on.""" - data = {registry.TAG_FIELD: "absent:Whatever"} with pytest.raises(InvalidModelTagError, match="Nothing registers"): - registry.resolve_tagged_model(data) + registry.resolve_tagged_model("absent:Whatever") def test_an_untagged_document_without_a_default_raises(self): """Nothing but the document says what a standalone document holds.""" with pytest.raises(InvalidModelTagError, match="declares no"): - registry.resolve_tagged_model({"a": 1}) + registry.resolve_tagged_model(None) def test_an_untagged_document_takes_the_default(self): """A caller which names the class does not need the document to.""" - assert registry.resolve_tagged_model({}, default=PatchAttrs) is PatchAttrs + assert registry.resolve_tagged_model(None, default=PatchAttrs) is PatchAttrs def test_a_broken_plugin_does_not_break_resolution(self, monkeypatch): """A plugin which cannot be imported has no models to find.""" @@ -396,10 +501,17 @@ def test_a_python_dump_is_untagged(self): assert registry.TAG_FIELD not in PatchAttrs().model_dump() assert registry.TAG_FIELD not in PatchAttrs().new(tag="a").model_dump() - def test_equality_is_unaffected(self, clean_registry): - """Two classes' instances are still told apart by their fields.""" - assert PatchAttrs(tag="a") == PatchAttrs(tag="a") - assert PatchAttrs(tag="a") != PatchAttrs(tag="b") + def test_include_and_exclude_still_filter_fields(self): + """ + A model serializer must not cost the caller `include`/`exclude`. + + Declaring it with `when_used="json"` reads better and silently + breaks them: pydantic skips the wrapper in python mode, and the + field filtering goes with it. `Patch.equals` dumps with `include`. + """ + attrs = PatchAttrs(tag="a") + assert set(attrs.model_dump(include={"tag"})) == {"tag"} + assert "tag" not in attrs.model_dump(exclude={"tag"}) def test_nested_models_state_their_class(self): """Universal, so a nested object can be read on its own later.""" @@ -424,19 +536,50 @@ def test_an_unknown_tag_is_accepted(self): attrs = PatchAttrs(**{registry.TAG_FIELD: "absent:Whatever"}) assert isinstance(attrs, PatchAttrs) + @pytest.mark.parametrize("value", ["my_sensor", "a b c", 3]) + def test_an_attr_which_names_no_class_is_kept(self, value): + """ + A reader's own metadata spelled like the tag is data, not a tag. + + PatchAttrs keeps extra fields, so consuming a value which names no + class would lose it silently; a tag DASCore wrote always resolves. + """ + attrs = PatchAttrs(**{registry.TAG_FIELD: value}) + assert attrs.model_dump()[registry.TAG_FIELD] == value + def test_the_tag_does_not_become_an_extra_field(self): """PatchAttrs keeps extras, and the tag is not one of them.""" attrs = PatchAttrs(**{registry.TAG_FIELD: "PatchAttrs"}) assert not hasattr(attrs, registry.TAG_FIELD) - def test_a_union_member_states_it_once(self): + def test_a_union_member_writes_its_own_tag(self, clean_registry): """ - The five resource models declare the tag as a real field. + The nine union members declare the tag as a real field. Pydantic must pick a class before an object exists, so their tag - cannot be a serializer concern; the base class leaves them alone - rather than writing a second copy of what they already wrote. + cannot be a serializer concern. The base class must leave the value + alone rather than overwrite it with the registry's name for the + class: the two differ for a subclass declared out of tree, and the + closed Literal would refuse to read the registry's spelling back. + """ + dumped = json.loads(Cable(resource_id="c1").model_dump_json()) + assert dumped[registry.TAG_FIELD] == "Cable" + # A plugin's subclass of a union member: tag and Literal disagree. + plugin_cable = type("PluginCable", (Cable,), {"__module__": "myplugin.cables"}) + assert registry.get_model_tag(plugin_cable) == "myplugin:PluginCable" + sub_dump = json.loads(plugin_cable(resource_id="c1").model_dump_json()) + assert sub_dump[registry.TAG_FIELD] == "Cable" + # Which is what keeps the document readable at all. + assert isinstance(Cable(**sub_dump), Cable) + + def test_a_union_member_states_its_tag_under_exclude_defaults(self): + """ + The tag is defaulted, so a dump flag would otherwise drop it. + + The serializer puts it back: a document which cannot say which union + member it holds cannot be read at all. """ - text = Cable(resource_id="c1").model_dump_json() - assert json.loads(text)[registry.TAG_FIELD] == "Cable" - assert text.count(f'"{registry.TAG_FIELD}"') == 1 + inventory = Inventory(resources={"c1": Cable(resource_id="c1")}) + dumped = inventory.model_dump(mode="json", exclude_defaults=True) + assert dumped["resources"]["c1"][registry.TAG_FIELD] == "Cable" + assert Inventory(**dumped) == inventory From cd87ce9e09102d5e13b48beb587d5778a1f63d5c Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Thu, 13 Aug 2026 13:38:41 +0200 Subject: [PATCH 8/8] Deselect the subprocess test in wasm and cover the shim WebAssembly has no subprocesses, and the wasm suite deselects the marker which says a test spawns one. The compat shim had no coverage because nothing in the repo imports it any more -- which is the point of it, and also why nothing was checking that it still re-exports what it promises. --- tests/test_models.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_models.py b/tests/test_models.py index 41bd0d78a..c150471be 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -8,9 +8,11 @@ import sys import numpy as np +import pydantic import pytest from pydantic import Field, ValidationError +import dascore.models as dc_models from dascore.core.attrs import PatchAttrs from dascore.core.inventory import Cable, Inventory from dascore.exceptions import InvalidModelTagError @@ -27,6 +29,7 @@ sensible_model_equals, ) from dascore.units import Quantity +from dascore.utils import models as models_shim class _TestModel(DascoreBaseModel): @@ -338,6 +341,30 @@ def test_optional_numbers_are_declared_across_the_formats(): assert total >= 25 +class TestDeprecatedModelPath: + """`dascore.utils.models` keeps working for readers which import it.""" + + def test_names_resolve_to_their_new_home(self): + """The same objects, not copies of them.""" + assert models_shim.DascoreBaseModel is DascoreBaseModel + assert models_shim.DateTime64 is DateTime64 + + def test_it_re_exports_everything_the_package_does(self): + """ + Re-exported wholesale, so a name added later cannot go missing. + + Which is the point of the star import: a second hand-maintained + list would drift from the first without anything noticing. + """ + assert set(dc_models.__all__) <= set(models_shim.__all__) + for name in models_shim.__all__: + assert hasattr(models_shim, name), name + + def test_it_still_carries_pydantic_base_model(self): + """It re-exported this, so something out of tree may import it.""" + assert models_shim.BaseModel is pydantic.BaseModel + + class TestModelTagRegistry: """A tag names one class, and the registry is what resolves it.""" @@ -465,6 +492,7 @@ def _boom(): ) assert registry.resolve_model_tag("absent:Whatever") is None + @pytest.mark.concurrency def test_a_model_in_an_unimported_module_is_found(self): """ A format's models only exist once its module is imported.