diff --git a/dascore/__init__.py b/dascore/__init__.py index 491f97617..814348f56 100644 --- a/dascore/__init__.py +++ b/dascore/__init__.py @@ -9,7 +9,8 @@ from dascore.core.attrs import PatchAttrs from dascore.core.summary import PatchSummary from dascore.core.spool import BaseSpool, Spool, spool -from dascore.core.inventory import Inventory, inventory +from dascore.core.inventory import Inventory +from dascore.core.inventory_loader import inventory from dascore.core.coordmanager import get_coord_manager, CoordManager from dascore.core.coords import get_coord from dascore.config import ( diff --git a/dascore/core/inventory.py b/dascore/core/inventory.py index 46511204e..47fae6b60 100644 --- a/dascore/core/inventory.py +++ b/dascore/core/inventory.py @@ -2307,29 +2307,3 @@ def from_yaml(cls, source) -> Self: msg = f"Could not parse an inventory mapping from {source!r}." raise InvalidInventoryError(msg) return cls(**data).check() - - -def inventory(source=None) -> Inventory: - """ - Load or create a DASDAE inventory. - - Parameters - ---------- - source - An existing Inventory (returned as is), a YAML file path or YAML - text, or None for an empty inventory. - - Examples - -------- - >>> import dascore as dc - >>> empty = dc.inventory() - >>> assert dc.inventory(empty) is empty - """ - if source is None: - return Inventory() - if isinstance(source, Inventory): - return source - if isinstance(source, str | os.PathLike): - return Inventory.from_yaml(source) - msg = f"Could not get an inventory from {source!r}." - raise InvalidInventoryError(msg) diff --git a/dascore/core/inventory_loader.py b/dascore/core/inventory_loader.py new file mode 100644 index 000000000..9c023f0d5 --- /dev/null +++ b/dascore/core/inventory_loader.py @@ -0,0 +1,921 @@ +""" +Load a DASDAE inventory from an authoring directory. + +The authoring format splits an inventory along its natural grain: small +heterogeneous objects (acquisitions, interrogators, cables) live in YAML or +JSON files matching the models, while long row-shaped track data lives in +CSV files a field crew can maintain as a spreadsheet. A directory of these +files is itself a loadable inventory, and ``to_yaml`` exports the +single-file interchange artifact for shipping beside a data archive. + +This module reads the object half. The track tables, and the optical path +epoch directories which hold them, are refused by name rather than +skipped, so a directory cannot load as an entity silently missing the +tracks its own files state. + +The contract, in one line: **file declares type, container agrees, name +implies identity, envelope implies version.** Every object file states what +it is, its container checks that statement rather than supplying it, its +name decides which entity it is, and the top-level ``inventory.yaml`` +versions the whole document. An address restated inside a file must agree +with the name; there is never a precedence rule between two spellings of +one fact. + +Loading is strict about near-misses and indifferent to clean misses: +anything which claims to participate in a convention and gets it wrong +raises, while anything which does not participate -- photos, field notes, +deployment logs -- is ignored where it lies. +""" + +from __future__ import annotations + +import json +import os +import re +from collections import defaultdict +from collections.abc import Mapping +from pathlib import Path +from typing import Any, NamedTuple + +import pandas as pd + +from dascore.core.inventory import ( + Acquisition, + Cable, + Enclosure, + ExternalResource, + FiberArray, + Interrogator, + Inventory, + Network, + OpticalMeasurement, + Station, + _overlapping_epochs, +) +from dascore.exceptions import ( + InvalidInventoryError, + MissingOptionalDependencyError, +) +from dascore.utils.misc import check_code, optional_import +from dascore.utils.models import InventoryModel, TimeRangedModel +from dascore.utils.time import to_datetime64 + +# One data model stands behind all three spellings, so they are accepted +# identically -- but one identity may only be spelled once. +_OBJECT_SUFFIXES = (".yaml", ".yml", ".json") + +# The object file of an entity directory, and the envelope at the root. +_ATTRS_STEM = "attrs" +_ENVELOPE_STEM = "inventory" + +# Separates an entity's name from the epoch it starts. +_EPOCH_MARKER = "@" + +# The reserved container stem: unlike an attribute table, these directories +# address a child entity whose name carries its epoch. +_PATH_STEM = "path" + +# The document-level collections, which live in the directory structure +# rather than in the envelope. +_ENVELOPE_COLLECTIONS = ("networks", "resources") + +# Address levels which name a containing entity rather than a field of the +# entity being named. +_ADDRESS_LEVELS = ("network", "fiber_array") + + +class _Container(NamedTuple): + """How one top-level directory maps entry names onto models.""" + + models: tuple[type[InventoryModel], ...] + # The dotted tokens an entry name holds, outermost first. A token which + # names a field of the model states that field; the rest are + # _ADDRESS_LEVELS naming the entity which contains it. + identity: tuple[str, ...] + # Collections whose members are addressed by their own names, in their + # own container. A file here may not state them: assembly would replace + # whatever it said, so stating them is a fact the format cannot keep. + supplied: tuple[str, ...] = () + + @property + def epochs(self) -> bool: + """Whether names here may carry an epoch, which needs a time range.""" + return all(issubclass(x, TimeRangedModel) for x in self.models) + + +_CONTAINERS: Mapping[str, _Container] = { + "resources": _Container( + (Interrogator, Cable, Enclosure, ExternalResource, OpticalMeasurement), + ("resource_id",), + ), + "networks": _Container((Network,), ("code",), ("fiber_arrays", "stations")), + "fiber_arrays": _Container((FiberArray,), ("network", "code"), ("acquisitions",)), + "stations": _Container((Station,), ("network", "code")), + "acquisitions": _Container( + (Acquisition,), ("network", "fiber_array", "location_code", "code") + ), +} + + +class _Entry(NamedTuple): + """One loaded entity, with where it came from and what contains it.""" + + # A resource or one of the time-ranged entities a network contains. + # Loosely typed because those two halves share only their base class, + # while assembling needs the fields of whichever half it holds. + model: Any + source: Path + # This entry's _ADDRESS_LEVELS tokens, outermost first. + address: tuple[str, ...] + + +def _model_names() -> frozenset[str]: + """ + Return the name of every inventory model. + + A file declaring one of these is claiming to be part of an inventory, + which is what makes it a near-miss rather than field material when it + turns up somewhere unrecognized. + """ + + def walk(model): + # Scoped to dascore's own models, so that what a caller happens to + # have subclassed and imported cannot change which files this + # format calls a near-miss. + if model.__module__.startswith("dascore."): + yield model.__name__ + for sub in model.__subclasses__(): + yield from walk(sub) + + return frozenset(walk(InventoryModel)) | {Inventory.__name__} + + +def _quote(path: Path) -> str: + """ + Name a file for an error message. + + Its container is included because a bare name is ambiguous across + containers and the full path is noise the reader already knows. + """ + return str(Path(path.parent.name) / path.name) + + +def _object_suffix(path: Path) -> str | None: + """ + Return a path's object-file suffix, or None if it has none. + + Matched without regard to case: a shouted ``DAS.L001.YAML`` is the file + ``DAS.L001.yaml`` would be, and skipping it for its spelling would load + an inventory silently missing whatever it named. + """ + suffix = path.suffix.casefold() + return suffix if suffix in _OBJECT_SUFFIXES else None + + +def _entry_name(path: Path) -> str: + """ + Return the address an entry's name states. + + ``Path.stem`` cannot be used: an address is full of dots, so it would + read ``DAS.L001`` as the stem ``DAS`` with a suffix. + """ + if path.is_dir() or (suffix := _object_suffix(path)) is None: + return path.name + return path.name[: -len(suffix)] + + +def _read_object(path: Path) -> dict[str, Any]: + """ + Parse one YAML or JSON object file into a mapping. + + Both spellings share one data model, so the suffix picks the parser + and decides nothing else. + """ + try: + text = path.read_text() + except (OSError, UnicodeDecodeError) as error: + msg = f"Could not read {_quote(path)}: {error}." + raise InvalidInventoryError(msg) from error + if _object_suffix(path) == ".json": + try: + data = json.loads(text) + except ValueError as error: + msg = f"Could not parse JSON from {_quote(path)}: {error}." + raise InvalidInventoryError(msg) from error + else: + yaml = optional_import("yaml", required_for="YAML inventory serialization") + try: + data = yaml.safe_load(text) + except yaml.YAMLError as error: + msg = f"Could not parse YAML from {_quote(path)}: {error}." + raise InvalidInventoryError(msg) from error + if not isinstance(data, Mapping): + msg = f"{_quote(path)} holds no mapping, so it defines no object." + raise InvalidInventoryError(msg) + return dict(data) + + +def _declared_type(path: Path) -> str | None: + """ + Return the model type a file declares, or None if it declares none. + + This tells field material from a misfiled object, so a file which does + not parse simply is not an object: a YAML-suffixed file under a photos + directory owes this format nothing. Nor does an unreadable one make an + inventory unloadable -- without PyYAML installed, a JSON inventory must + still load past whatever YAML happens to lie beside it. + """ + try: + data = _read_object(path) + except (InvalidInventoryError, MissingOptionalDependencyError): + return None + declared = data.get("type") + return declared if isinstance(declared, str) else None + + +# The date is extended ISO 8601 and the time is basic, because ':' is not a +# legal filename character on Windows. +_EPOCH_RE = re.compile(r"(?P\d{4}-\d{2}-\d{2})(?:T(?P