From 057475c4c522233d336a1b057e970eb81396d22e Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Thu, 13 Aug 2026 08:55:55 +0200 Subject: [PATCH 1/9] Read an inventory from an authoring directory An inventory can now be laid out as a directory of YAML or JSON object files, which dc.inventory loads: file declares type, container agrees, name implies identity, envelope implies version. Names are addresses, so the nested tree materializes from a flat directory and a network or fiber array mentioned only by an address exists. Track CSVs and optical path epoch directories are refused by name until they can be read, rather than loading an entity which silently lacks the tracks its own directory states. --- dascore/__init__.py | 3 +- dascore/core/inventory.py | 26 - dascore/core/inventory_loader.py | 715 +++++++++++++++++++++++ tests/test_core/test_inventory_loader.py | 674 +++++++++++++++++++++ 4 files changed, 1391 insertions(+), 27 deletions(-) create mode 100644 dascore/core/inventory_loader.py create mode 100644 tests/test_core/test_inventory_loader.py diff --git a/dascore/__init__.py b/dascore/__init__.py index 491f97617..814348f56 100644 --- a/dascore/__init__.py +++ b/dascore/__init__.py @@ -9,7 +9,8 @@ from dascore.core.attrs import PatchAttrs from dascore.core.summary import PatchSummary from dascore.core.spool import BaseSpool, Spool, spool -from dascore.core.inventory import Inventory, inventory +from dascore.core.inventory import Inventory +from dascore.core.inventory_loader import inventory from dascore.core.coordmanager import get_coord_manager, CoordManager from dascore.core.coords import get_coord from dascore.config import ( diff --git a/dascore/core/inventory.py b/dascore/core/inventory.py index 46511204e..47fae6b60 100644 --- a/dascore/core/inventory.py +++ b/dascore/core/inventory.py @@ -2307,29 +2307,3 @@ def from_yaml(cls, source) -> Self: msg = f"Could not parse an inventory mapping from {source!r}." raise InvalidInventoryError(msg) return cls(**data).check() - - -def inventory(source=None) -> Inventory: - """ - Load or create a DASDAE inventory. - - Parameters - ---------- - source - An existing Inventory (returned as is), a YAML file path or YAML - text, or None for an empty inventory. - - Examples - -------- - >>> import dascore as dc - >>> empty = dc.inventory() - >>> assert dc.inventory(empty) is empty - """ - if source is None: - return Inventory() - if isinstance(source, Inventory): - return source - if isinstance(source, str | os.PathLike): - return Inventory.from_yaml(source) - msg = f"Could not get an inventory from {source!r}." - raise InvalidInventoryError(msg) diff --git a/dascore/core/inventory_loader.py b/dascore/core/inventory_loader.py new file mode 100644 index 000000000..f67f3688e --- /dev/null +++ b/dascore/core/inventory_loader.py @@ -0,0 +1,715 @@ +""" +Load a DASDAE inventory from an authoring directory. + +The authoring format splits an inventory along its natural grain: small +heterogeneous objects (acquisitions, interrogators, cables) live in YAML or +JSON files matching the models, while long row-shaped track data lives in +CSV files a field crew can maintain as a spreadsheet. A directory of these +files is itself a loadable inventory, and ``to_yaml`` exports the +single-file interchange artifact for shipping beside a data archive. + +The contract, in one line: **file declares type, container agrees, name +implies identity, envelope implies version.** Every object file states what +it is, its container checks that statement rather than supplying it, its +name decides which entity it is, and the top-level ``inventory.yaml`` +versions the whole document. An address restated inside a file must agree +with the name; there is never a precedence rule between two spellings of +one fact. + +Loading is strict about near-misses and indifferent to clean misses: +anything which claims to participate in a convention and gets it wrong +raises, while anything which does not participate -- photos, field notes, +deployment logs -- is ignored where it lies. +""" + +from __future__ import annotations + +import json +import os +import re +from collections import defaultdict +from collections.abc import Mapping +from pathlib import Path +from typing import Any, NamedTuple + +import pandas as pd + +from dascore.core.inventory import ( + Acquisition, + Cable, + Enclosure, + ExternalResource, + FiberArray, + Interrogator, + Inventory, + Network, + OpticalMeasurement, + Station, +) +from dascore.exceptions import InvalidInventoryError +from dascore.utils.misc import optional_import +from dascore.utils.models import InventoryModel, TimeRangedModel +from dascore.utils.time import to_datetime64 + +# One data model stands behind all three spellings, so they are accepted +# identically -- but one identity may only be spelled once. +_OBJECT_SUFFIXES = (".yaml", ".yml", ".json") + +# The object file of an entity directory, and the envelope at the root. +_ATTRS_STEM = "attrs" +_ENVELOPE_STEM = "inventory" + +# Separates an entity's name from the epoch it starts. +_EPOCH_MARKER = "@" + +# The reserved container stem: unlike an attribute table, these directories +# address a child entity whose name carries its epoch. +_PATH_STEM = "path" + +# The document-level collections, which live in the directory structure +# rather than in the envelope. +_ENVELOPE_COLLECTIONS = ("networks", "resources") + +# Address levels which name a containing entity rather than a field of the +# entity being named. +_ADDRESS_LEVELS = ("network", "fiber_array") + + +class _Container(NamedTuple): + """How one top-level directory maps entry names onto models.""" + + models: tuple[type[InventoryModel], ...] + # The dotted tokens an entry name holds, outermost first. A token which + # names a field of the model states that field; the rest are + # _ADDRESS_LEVELS naming the entity which contains it. + identity: tuple[str, ...] + + +_CONTAINERS: Mapping[str, _Container] = { + "resources": _Container( + (Interrogator, Cable, Enclosure, ExternalResource, OpticalMeasurement), + ("resource_id",), + ), + "networks": _Container((Network,), ("code",)), + "fiber_arrays": _Container((FiberArray,), ("network", "code")), + "stations": _Container((Station,), ("network", "code")), + "acquisitions": _Container( + (Acquisition,), ("network", "fiber_array", "location_code", "code") + ), +} + + +class _Entry(NamedTuple): + """One loaded entity, with where it came from and what contains it.""" + + model: InventoryModel + source: Path + # This entry's _ADDRESS_LEVELS tokens, outermost first. + address: tuple[str, ...] + + +def _model_names() -> frozenset[str]: + """ + Return the name of every inventory model. + + A file declaring one of these is claiming to be part of an inventory, + which is what makes it a near-miss rather than field material when it + turns up somewhere unrecognized. + """ + + def walk(model): + yield model.__name__ + for sub in model.__subclasses__(): + yield from walk(sub) + + return frozenset(walk(InventoryModel)) | {Inventory.__name__} + + +def _quote(path: Path) -> str: + """ + Name a file for an error message. + + Its container is included because a bare name is ambiguous across + containers and the full path is noise the reader already knows. + """ + return str(Path(path.parent.name) / path.name) + + +def _entry_name(path: Path) -> str: + """ + Return the address an entry's name states. + + ``Path.stem`` cannot be used: an address is full of dots, so it would + read ``DAS.L001`` as the stem ``DAS`` with a suffix. + """ + if path.is_dir() or path.suffix not in _OBJECT_SUFFIXES: + return path.name + return path.name[: -len(path.suffix)] + + +def _read_object(path: Path) -> dict[str, Any]: + """ + Parse one YAML or JSON object file into a mapping. + + Both spellings share one data model, so the suffix picks the parser + and decides nothing else. + """ + try: + text = path.read_text() + except (OSError, UnicodeDecodeError) as error: + msg = f"Could not read {_quote(path)}: {error}." + raise InvalidInventoryError(msg) from error + if path.suffix == ".json": + try: + data = json.loads(text) + except ValueError as error: + msg = f"Could not parse JSON from {_quote(path)}: {error}." + raise InvalidInventoryError(msg) from error + else: + yaml = optional_import("yaml", required_for="YAML inventory serialization") + try: + data = yaml.safe_load(text) + except yaml.YAMLError as error: + msg = f"Could not parse YAML from {_quote(path)}: {error}." + raise InvalidInventoryError(msg) from error + if not isinstance(data, Mapping): + msg = f"{_quote(path)} holds no mapping, so it defines no object." + raise InvalidInventoryError(msg) + return dict(data) + + +def _declared_type(path: Path) -> str | None: + """ + Return the model type a file declares, or None if it declares none. + + This tells field material from a misfiled object, so a file which does + not parse simply is not an object: a YAML-suffixed file under a photos + directory owes this format nothing. + """ + try: + data = _read_object(path) + except InvalidInventoryError: + return None + declared = data.get("type") + return declared if isinstance(declared, str) else None + + +# The date is extended ISO 8601 and the time is basic, because ':' is not a +# legal filename character on Windows. +_EPOCH_RE = re.compile(r"(?P\d{4}-\d{2}-\d{2})(?:T(?P