diff --git a/dascore/core/inventory_loader.py b/dascore/core/inventory_loader.py index 33b26c705..d16b712e4 100644 --- a/dascore/core/inventory_loader.py +++ b/dascore/core/inventory_loader.py @@ -8,10 +8,11 @@ files is itself a loadable inventory, and ``to_yaml`` exports the single-file interchange artifact for shipping beside a data archive. -This module reads the object half. The track tables, and the optical path -epoch directories which hold them, are refused by name rather than -skipped, so a directory cannot load as an entity silently missing the -tracks its own files state. +A table is matched to the model by name: ``.csv`` fills the +attribute ```` of the type its directory declares. ``path`` is the +one reserved container stem -- those directories address an optical path +epoch rather than serializing an attribute, and each location code is a +lineage in which an epoch runs until its successor begins. The contract, in one line: **file declares object_type, container agrees, name implies identity, envelope implies version.** Every object file states what @@ -29,6 +30,8 @@ from __future__ import annotations +import csv +import itertools import json import os import re @@ -42,6 +45,7 @@ from dascore.core.inventory import ( Acquisition, Cable, + CoordinateReferenceSystem, Enclosure, ExternalResource, FiberArray, @@ -49,8 +53,10 @@ Inventory, Network, OpticalMeasurement, + OpticalPath, Station, _overlapping_epochs, + _times_equal, ) from dascore.exceptions import ( InvalidInventoryError, @@ -418,7 +424,7 @@ def _apply_identity(data: dict, container: _Container, name: str, source: Path): return tuple(address) -def _load_entry(entry: Path, data_source: Path, container: _Container) -> _Entry: +def _load_entry(entry: Path, data_source: Path, container: _Container, crs) -> _Entry: """Load one entry of a container from its object file.""" data = _read_object(data_source) model = _pick_model(data, container, data_source) @@ -427,6 +433,11 @@ def _load_entry(entry: Path, data_source: Path, container: _Container) -> _Entry address = _apply_identity(data, container, name, data_source) if epoch is not None and "start_time" not in data: data["start_time"] = epoch + # After the epoch, not before: an entity's first path epoch starts + # where the entity does, so its own start has to be known by then. + if entry.is_dir(): + _merge_tables(data, entry, model, crs, data_source) + _merge_paths(data, entry, model, crs, data_source, data.get("start_time")) built = _build(model, data, data_source) if epoch is not None and built.start_time != epoch: msg = ( @@ -438,30 +449,578 @@ def _load_entry(entry: Path, data_source: Path, container: _Container) -> _Entry return _Entry(built, data_source, address) -def _refuse_tracks(entity: Path) -> None: +class _Table(NamedTuple): + """How the rows of one track table map onto the attribute they fill.""" + + # False when a row is one object; True when a row is one control point + # of an object which holds parallel arrays. + points: bool = False + # The column assigning points to objects, for a collection of them. + # None where the attribute is a single object, or a row is an object. + group: str | None = None + # The column rows are read in the order of. Where a table names one, + # row position decides nothing and re-sorting a spreadsheet is + # harmless; where it does not, the rows keep the order they were + # written in, which the model reads as a set rather than a sequence. + order: str | None = None + # True where that column is the table's own scaffolding rather than a + # field, so nothing else records where a row sits and it must place + # each row unambiguously. + places: bool = False + + +# Keyed by CSV stem, which is the attribute the table fills. Held as a +# registry rather than introspected: how rows map onto objects is a +# property of the attribute, and stating it is shorter than deducing it. +# TestTableRegistry pins every key to a field of the model declaring it. +_TABLES: Mapping[str, _Table] = { + "optical_components": _Table(order="sequence", places=True), + "coupling": _Table(), + "annotations": _Table(), + "geometry": _Table(points=True, group="segment", order="distance"), + "distance_map": _Table(points=True, order="distance"), +} + +# The one column of a point table which is not a field of the object it +# builds; components order by it and drop it. +_SEQUENCE = "sequence" + + +def _read_table(path: Path) -> pd.DataFrame: + """ + Read one track table. + + Every cell arrives as text and the models coerce it, so a column's + meaning is the field's rather than whatever pandas inferred from the + rows it happened to see. Only a truly empty cell is null: an empty + cell means unset, and a document which writes ``NA`` means the string. + """ + # The header is read first and by itself, for two reasons: pandas + # renames a repeated column rather than refusing it, so by the time a + # frame exists the second one is `coupling_type.1` and the clash cannot + # be seen; and it raises its own error for a file with no columns, + # which would arrive before this one could say what was expected. + try: + with path.open(newline="", encoding="utf-8-sig") as stream: + reader = csv.reader(stream) + header = next(reader, []) + if header: + # Streamed rather than listed: a track table is the part of + # this format meant to grow, and holding every cell as a + # python object beside the frame pandas builds would cost + # several times what the frame itself does. + _check_widths(reader, header, path) + except (OSError, UnicodeDecodeError) as error: + msg = f"Could not read {_quote(path)}: {error}." + raise InvalidInventoryError(msg) from error + if not header: + msg = f"{_quote(path)} has no columns, so it states no track." + raise InvalidInventoryError(msg) + repeated = sorted({x for x in header if header.count(x) > 1}) + if repeated: + msg = ( + f"{_quote(path)} names {', '.join(repeated)} more than once; one " + "column states one field." + ) + raise InvalidInventoryError(msg) + # index_col=False so that no column is ever read as an index; the row + # widths above already agree, and this keeps them agreeing. + return pd.read_csv( + path, + dtype=str, + keep_default_na=False, + na_values=[""], + index_col=False, + # Both readers decode alike, or the header checked above is not + # the header parsed here: a locale-encoded read disagrees with + # pandas' UTF-8, and a byte order mark reaches only one of them. + encoding="utf-8-sig", + ) + + +def _check_widths(reader, header: list[str], path: Path) -> None: + """ + Refuse a row which is not its header wide. + + Pandas refuses neither a wide row nor a narrow one: by default the + surplus cell pushes the first column into the index, so every value in + the row shifts one field left and lands in its neighbour's meaning. A + row states one cell per column or it is not a row. + """ + for number, row in enumerate(reader, start=2): + if row and len(row) != len(header): + msg = ( + f"{_quote(path)} row {number} states {len(row)} cells where " + f"its header names {len(header)} columns." + ) + raise InvalidInventoryError(msg) + + +def _cells(row) -> dict[str, str]: + """Return a row's stated cells, an empty one meaning unset.""" + return {str(k): v for k, v in row.items() if not pd.isnull(v)} + + +def _require_columns(frame: pd.DataFrame, needed, path: Path) -> None: + """Refuse a table which does not carry a column it is read by.""" + missing = [x for x in needed if x is not None and x not in frame.columns] + if missing: + msg = ( + f"{_quote(path)} states no {', '.join(missing)} column, which its " + "rows are read by." + ) + raise InvalidInventoryError(msg) + + +def _require_stated(frame: pd.DataFrame, needed, path: Path) -> None: + """ + Refuse a blank cell in a column the table is read by. + + A column which orders or groups the rows decides where each one goes, + so a row leaving it empty has no place. Left to pandas the row would + simply disappear -- a null sorts last, and a null grouping key drops + its row from every group. + """ + for column in needed: + if column is None: + continue + empty = [ + str(n) for n, ok in enumerate(frame[column].notna(), start=2) if not ok + ] + if empty: + msg = ( + f"{_quote(path)} leaves {column} empty at row(s) " + f"{', '.join(empty)}, so those rows state no place." + ) + raise InvalidInventoryError(msg) + + +def _ordered(frame: pd.DataFrame, column: str | None, path: Path) -> pd.DataFrame: + """ + Return the rows in the order the named column states, if any. + + A table which names one is read by it rather than by row position, so + re-sorting a spreadsheet cannot change what it means. A table which + names none keeps the order it was written in. + """ + if column is None: + return frame + try: + keys = pd.to_numeric(frame[column]) + except (TypeError, ValueError) as error: + msg = f"{_quote(path)} has a non-numeric {column}: {error}." + raise InvalidInventoryError(msg) from error + return frame.assign(**{column: keys}).sort_values(column, kind="stable") + + +def _parse_cell(text: str): + """ + Read a cell's value the way its own text states it. + + A CSV has no types, so an annotation's value -- which the model lets + be a string, a boolean or a number -- is decided by what was written. + A value which is genuinely a string but looks like one of the others + is the one thing this spelling cannot express; that group is authored + in YAML, where the types are explicit. + """ + if (folded := text.strip().casefold()) in ("true", "false"): + return folded == "true" + try: + number = float(text) + except ValueError: + return text + # int(number) rather than int(text): 1e3 is integral, and only the + # number knows that -- the text raises. + return int(number) if number.is_integer() and "." not in text else number + + +def _check_places(keys: pd.Series, column: str, path: Path) -> None: + """ + Refuse an ordering which does not place every row. + + Components tile the path, each starting where the previous ends, so + two rows sharing a place would be ordered by where they happen to sit + in the file -- which is the one thing this column exists to stop + deciding anything. + """ + repeated = sorted({str(x) for x in keys[keys.duplicated()]}) + if repeated: + msg = ( + f"{_quote(path)} states {column} {', '.join(repeated)} more than " + "once, so it does not say which row comes first." + ) + raise InvalidInventoryError(msg) + + +def _object_rows(frame: pd.DataFrame, table: _Table, path: Path) -> list[dict]: + """Read a table whose every row is one object.""" + _require_columns(frame, [table.order], path) + _require_stated(frame, [table.order], path) + ordered = _ordered(frame, table.order, path) + if table.places and table.order is not None: + _check_places(ordered[table.order], table.order, path) + out = [] + for _, row in ordered.iterrows(): + cells = _cells(row) + # The order column is the table's own scaffolding where the object + # has no such field, so it is dropped -- but only where the table + # says it has one. Dropped everywhere, a stray sequence column in + # coupling.csv would vanish instead of being refused as the + # unknown field the model calls it. + if table.places: + cells.pop(table.order, None) + out.append(cells) + return out + + +def _point_rows(frame: pd.DataFrame, table: _Table, path: Path, axes) -> list[dict]: + """ + Read a table whose every row is one control point. + + Points gather into objects holding parallel arrays: one object per + value of the grouping column, or a single object when the attribute + is one. Coordinate columns are named by the CRS and are stored on the + canonical axes, so the frame decides which column is which. + """ + _require_columns(frame, [table.order, table.group], path) + _require_stated(frame, [table.order, table.group], path) + frame = _ordered(frame, table.order, path) + # dropna=False: a blank grouping cell would otherwise take its row out + # of the table without a word. _require_stated has already refused one, + # and this keeps that the reason nothing is missing. + groups = ( + frame.groupby(table.group, sort=True, dropna=False) + if table.group + else [(None, frame)] + ) + out = [] + for name, rows in groups: + point: dict[str, Any] = {} if name is None else {"name": str(name)} + for column in rows.columns: + if column == table.group or column in axes: + continue + stated = rows[column].notna() + if not stated.any(): + continue + # Each column becomes one array and the arrays are read + # together, so a column stated by some rows and not others + # would compact past the gap and pair values which never + # shared a row: `5,` above `,100` would map channel 5 to a + # distance the file never gave it. + if not stated.all(): + # rows.index, not the position in this group: the frame has + # been sorted and split by then, so counting here would name + # a line the reader would go and find something else on. + empty = [ + str(i + 2) + for i, ok in zip(rows.index, stated, strict=True) + if not ok + ] + msg = ( + f"{_quote(path)} leaves {column} empty at row(s) " + f"{', '.join(empty)} while other rows state it; a column " + "is stated by every point or by none." + ) + raise InvalidInventoryError(msg) + point[column] = tuple(rows[column]) + if axes: + point["coordinates"] = _coordinates(rows, axes, path) + out.append(point) + return out + + +def _coordinates(rows: pd.DataFrame, axes: Mapping[str, int], path: Path): + """ + Gather a geometry table's labelled columns onto the canonical axes. + + The CRS names the axes and states their order, so a header is read by + which axis it names rather than by where it sits in the file. + """ + ordered = sorted(axes, key=lambda label: axes[label]) + out = [] + for _, row in rows.iterrows(): + stated = [row[label] for label in ordered] + if any(pd.isnull(x) for x in stated): + missing = [x for x, v in zip(ordered, stated, strict=True) if pd.isnull(v)] + msg = ( + f"{_quote(path)} leaves {', '.join(missing)} empty for a point; " + "a coordinate states every axis its frame declares." + ) + raise InvalidInventoryError(msg) + out.append(tuple(stated)) + return tuple(out) + + +def _is_path_dir(child: Path) -> bool: + """Return True if a directory name claims to be an optical path epoch.""" + # is_dir() follows a link, and the stray walk steps over one, so a + # symlinked `path` would be read from outside the inventory without + # anything having looked at what it holds. + if not child.is_dir() or child.is_symlink() or child.name.startswith("."): + return False + stem = child.name.partition(_EPOCH_MARKER)[0].partition(".")[0] + return stem.casefold() == _PATH_STEM + + +def _check_one_spelling(directories: list[Path]) -> None: + """ + Refuse two path directories a case-folding filesystem holds as one. + + The same portability rule `_container_entries` applies to every + top-level identity: an inventory where `path.aa` and `path.AA` are two + lineages loses one of them the moment it is copied somewhere they are + the same directory. + """ + seen: dict[str, Path] = {} + for directory in directories: + key = directory.name.casefold() + if (first := seen.get(key)) is not None: + msg = ( + f"{_quote(first)} and {_quote(directory)} differ only by case, " + "which a case-insensitive filesystem cannot hold." + ) + raise InvalidInventoryError(msg) + seen[key] = directory + + +def _load_path(directory: Path, crs, begins): + """ + Read one optical path epoch from its own directory. + + ``path`` is the one reserved container stem: unlike an attribute + table, these directories do not serialize an attribute of the fiber + array -- they address a child entity whose name carries its location + and the instant it starts. + """ + name, epoch = _split_epoch(directory, epochs_allowed=True) + _, _, location = name.partition(".") + attrs = _attrs_file(directory) + data = _read_object(attrs) + declared = data.get(TAG_FIELD) + if declared != OpticalPath.__name__: + msg = ( + f"{_quote(attrs)} declares {declared!r}, but a {_PATH_STEM} " + f"directory holds an {OpticalPath.__name__}." + ) + raise InvalidInventoryError(msg) + stated = data.setdefault("location_code", location) + if stated != location: + msg = ( + f"{_quote(attrs)} states location_code={stated!r} but its " + f"directory says {location!r}. A restated address must agree " + "with the name." + ) + raise InvalidInventoryError(msg) + if epoch is not None and "start_time" not in data: + data["start_time"] = epoch + # The bare `path` directory is the first epoch, and it starts where the + # fiber array holding it does -- left unset it would claim the + # unbounded past, which is before the array it belongs to exists. + if epoch is None and "start_time" not in data and not pd.isnull(begins): + data["start_time"] = begins + _merge_tables(data, directory, OpticalPath, crs, attrs) + built = _build(OpticalPath, data, attrs) + if epoch is not None and built.start_time != epoch: + msg = ( + f"{_quote(attrs)} states start_time {built.start_time} but its " + f"directory says {epoch}. A restated address must agree with " + "the name." + ) + raise InvalidInventoryError(msg) + return built + + +def _close_lineages(paths: list, sources: dict) -> list: """ - Refuse the parts of an entity directory which cannot be read yet. + End each epoch where the next one of its lineage begins. - Track tables and optical path epochs are the next piece of this - format. Refusing them by name beats loading an entity which silently - lacks the tracks its own directory states. + Each location code is its own lineage, non-overlapping by + construction: sorted by start, an epoch runs until its successor and + the last is ongoing. An epoch may state an earlier end itself -- a + dark interval, or a retired lineage -- but not a later one, which + would claim time its successor already holds. + """ + out = [] + by_location = defaultdict(list) + for path in paths: + by_location[path.location_code].append(path) + for location, lineage in by_location.items(): + # An unset start is the unbounded past, so the bare `path` directory + # sorts before every epoch which names an instant, rather than after + # them as a null ordinarily would. + ordered = sorted( + lineage, key=lambda x: (not pd.isnull(x.start_time), x.start_time) + ) + for first, second in itertools.pairwise(ordered): + # _times_equal, not ==: NaT equals nothing, itself included, + # so two undated epochs of one lineage would never collide. + if _times_equal(first.start_time, second.start_time): + msg = ( + f"{_quote(sources[id(first)])} and " + f"{_quote(sources[id(second)])} start at the same instant, " + "so they are two spellings of one epoch." + ) + raise InvalidInventoryError(msg) + if pd.isnull(first.end_time): + out.append(first.new(end_time=second.start_time)) + continue + if first.end_time > second.start_time: + msg = ( + f"{_quote(sources[id(first)])} ends at {first.end_time}, " + f"after the epoch which follows it begins at " + f"{second.start_time}." + ) + raise InvalidInventoryError(msg) + out.append(first) + out.append(ordered[-1]) + return out + + +def _merge_paths(data: dict, entity: Path, model, crs, attrs: Path, begins) -> None: + """Fill the optical paths an entity directory's epoch directories state.""" + directories = [x for x in sorted(entity.iterdir()) if _is_path_dir(x)] + if not directories: + return + _check_one_spelling(directories) + field = "optical_paths" + if field not in model.model_fields: + msg = ( + f"{_quote(directories[0])} is an optical path epoch, but " + f"{_quote(attrs)} declares a {model.__name__}, which holds none." + ) + raise InvalidInventoryError(msg) + if field in data: + msg = ( + f"{field} is stated both in {_quote(attrs)} and as " + f"{_PATH_STEM} directories; one fact is spelled once." + ) + raise InvalidInventoryError(msg) + paths, sources = [], {} + for directory in directories: + built = _load_path(directory, crs, begins) + # The file itself, not a name built from the stem: the suffix is + # whichever of the three the author used, and an error naming + # `path/attrs` sends them looking for a file which is not there. + sources[id(built)] = _attrs_file(directory) + paths.append(built) + data[field] = _close_lineages(paths, sources) + + +def _table_stem(path: Path) -> str: + """Return the attribute a table's name states.""" + return path.name[: -len(path.suffix)] + + +def _merge_tables(data: dict, entity: Path, model, crs, attrs: Path) -> None: + """ + Fill the attributes an entity directory's tables state. + + A table is matched to the model purely by name, so a stem which names + no attribute of the declared type is a typo rather than a new track, + and an attribute stated both inline and as a table is one fact spelled + twice. """ for child in sorted(entity.iterdir()): - if child.name.startswith("."): + if child.name.startswith(".") or child.is_dir(): + continue + if child.suffix.casefold() != ".csv": continue - stem = child.name.partition(_EPOCH_MARKER)[0].partition(".")[0].casefold() - if child.is_dir() and stem == _PATH_STEM: + stem = _table_stem(child) + if stem not in model.model_fields: msg = ( - f"{_quote(child)} is an optical path epoch, which cannot be " - "read yet. State optical_paths in the entity's " - f"{_ATTRS_STEM} file for now." + f"{_quote(child)} names no attribute of {model.__name__}, which " + f"{_quote(attrs)} declares." ) raise InvalidInventoryError(msg) - if child.suffix.casefold() == ".csv": + if (table := _TABLES.get(stem)) is None: + # Not "is not row-shaped": Station.channels is as row-shaped as + # anything here and still has no table, so saying that would be + # telling the author something false about their own model. msg = ( - f"{_quote(child)} is a track table, which cannot be read yet. " - f"State {_entry_name(child)} in the entity's {_ATTRS_STEM} " - "file for now." + f"{_quote(child)} names {stem}, which this format does not " + f"read as a table; state it in the {_ATTRS_STEM} file instead." + ) + raise InvalidInventoryError(msg) + if stem in data: + msg = ( + f"{stem} is stated both in {_quote(attrs)} and as " + f"{_quote(child)}; one fact is spelled once." + ) + raise InvalidInventoryError(msg) + data[stem] = _load_table(child, table, stem, crs) + + +def _load_table(path: Path, table: _Table, stem: str, crs): + """Read one track table into whatever its attribute holds.""" + frame = _read_table(path) + # Refused here rather than left to the model: a header with nothing + # under it claims a track and states none, and for a single-object + # table it would otherwise build one object out of no points. + if frame.empty: + msg = f"{_quote(path)} states no rows, so it describes no {stem}." + raise InvalidInventoryError(msg) + axes = _geometry_axes(frame, crs, path) if stem == "geometry" else {} + if not table.points: + rows = _object_rows(frame, table, path) + if stem == "annotations": + _parse_annotations(rows, path) + return rows + built = _point_rows(frame, table, path, axes) + # A single object rather than a collection: the table has no grouping + # column because every point belongs to the one map it describes. + return built if table.group is not None else built[0] + + +def _geometry_axes(frame: pd.DataFrame, crs, path: Path) -> dict[str, int]: + """ + Return which column names which canonical axis. + + Coordinates are stored on the canonical axes while a geometry table + names them the way its frame does, so the CRS decides both which + headers are legal and what each one means. + """ + labels = tuple(crs.coordinate_labels) + stated = {x for x in frame.columns} - {"segment", "distance"} + if stated != set(labels): + msg = ( + f"{_quote(path)} states the coordinate columns {sorted(stated)}, " + f"but its frame declares {list(labels)}." + ) + raise InvalidInventoryError(msg) + return {label: index for index, label in enumerate(labels)} + + +def _parse_annotations(rows: list[dict], path: Path) -> None: + """ + Read each annotation's value as its own text states it, in place. + + A group's kind is decided by its values, and the model makes the kind + decide the group's shape, so a group which mixes kinds would be two + tracks sharing a name. + """ + kinds: dict[str, tuple[str, int]] = {} + for number, row in enumerate(rows, start=2): + if (text := row.get("value")) is None: + continue + row["value"] = value = _parse_cell(text) + # A boolean is asked about first because a bool IS an int, which + # would otherwise let true and 1 share a group whose shape they do + # not share. An int and a float, by contrast, are ONE kind: the + # model reads them alike, so telling them apart here would make + # the order the rows were written in decide whether a group loads. + kind = ( + "a boolean" + if isinstance(value, bool) + else "a number" + if isinstance(value, int | float) + else "text" + ) + group = str(row.get("group", "")) + first, where = kinds.setdefault(group, (kind, number)) + if first != kind: + msg = ( + f"{_quote(path)} row {number}: group {group!r} states {kind} " + f"where row {where} states {first}; one group holds one kind." ) raise InvalidInventoryError(msg) @@ -561,20 +1120,19 @@ def _container_entries(directory: Path) -> list[Path]: return list(seen.values()) -def _load_container(directory: Path, container: _Container, root: Path): +def _load_container(directory: Path, container: _Container, root: Path, crs): """Load every entry of one top-level container directory.""" out = [] for child in _container_entries(directory): if child.is_dir(): - _refuse_tracks(child) data_source = _attrs_file(child) # An entity directory holds its own attrs and tracks; an object # filed inside it belongs to a container and is not loaded from # here, so it has to be refused rather than stepped over. - _refuse_stray_objects(child, root, skip=data_source) + _refuse_stray_objects(child, root, skip=_contained_files(child)) else: data_source = child - out.append(_load_entry(child, data_source, container)) + out.append(_load_entry(child, data_source, container, crs)) return out @@ -811,7 +1369,22 @@ def _load_envelope(root: Path) -> dict[str, Any] | None: return data -def _refuse_stray_objects(start: Path, root: Path, skip: Path | None = None) -> None: +def _contained_files(entity: Path) -> frozenset[Path]: + """ + Return the object files the format itself places inside an entity. + + Its own attrs file, and the attrs file of each optical path epoch, + which is an entity in its own right rather than an object filed where + nothing holds it. + """ + out = {_attrs_file(entity)} + for child in sorted(entity.iterdir()): + if _is_path_dir(child): + out.add(_attrs_file(child)) + return frozenset(out) + + +def _refuse_stray_objects(start: Path, root: Path, skip=frozenset()) -> None: """ Refuse a model-declaring file which nothing contains. @@ -823,7 +1396,7 @@ def _refuse_stray_objects(start: Path, root: Path, skip: Path | None = None) -> known = _model_names() def check(path: Path): - if path.name.startswith(".") or path == skip: + if path.name.startswith(".") or path in skip: return # A symlink is not part of the format, and one pointing at an # ancestor walks the inventory a second time -- where its own files @@ -847,6 +1420,21 @@ def check(path: Path): check(start) +def _build_crs(envelope: dict[str, Any] | None): + """ + Return the frame the document declares, or the implicit default. + + Built before anything else is read, since a geometry table's headers + are legal or not according to what this states. Whatever the envelope + states here already built once, when the envelope was read as a whole, + so an unreadable frame is reported there rather than again here. + """ + stated = (envelope or {}).get("coordinate_reference_system") + return ( + CoordinateReferenceSystem(**stated) if stated else CoordinateReferenceSystem() + ) + + def _check_strays(root: Path) -> None: """Refuse a stray object anywhere outside a recognized container.""" for child in sorted(root.iterdir()): @@ -868,8 +1456,11 @@ def load_directory(path: str | os.PathLike) -> Inventory: """ root = Path(path) envelope = _load_envelope(root) + # A geometry table names its columns the way its frame does, so the + # envelope is read first: it is what says which names those are. + crs = _build_crs(envelope) entries = { - name: _load_container(root / name, container, root) + name: _load_container(root / name, container, root, crs) for name, container in _CONTAINERS.items() if (root / name).is_dir() } diff --git a/tests/test_core/test_inventory_loader.py b/tests/test_core/test_inventory_loader.py index 3f0feb56f..fc8e66a12 100644 --- a/tests/test_core/test_inventory_loader.py +++ b/tests/test_core/test_inventory_loader.py @@ -7,6 +7,7 @@ from pathlib import Path import numpy as np +import pandas as pd import pytest from pydantic import ValidationError @@ -51,6 +52,22 @@ def _folds_case() -> bool: FOLDS_CASE = _folds_case() +def _keeps_trailing_dot() -> bool: + """Return True if this filesystem keeps a trailing dot in a name.""" + with tempfile.TemporaryDirectory() as name: + directory = Path(name) + try: + (directory / "probe.").mkdir() + except OSError: # pragma: no cover - no filesystem here refuses it + return False + return (directory / "probe.").exists() and not (directory / "probe").exists() + + +# Windows strips a trailing dot, so `path.` and `path` are one directory +# there and the blank location cannot be spelled twice. +KEEPS_TRAILING_DOT = _keeps_trailing_dot() + + @pytest.fixture def make_inventory(tmp_path): """Return a function which writes a directory and loads it.""" @@ -791,27 +808,7 @@ def test_unreadable_file_names_itself(self, tmp_path): class TestSeams: - """The parts of the format which cannot be read yet are refused.""" - - def test_track_table_in_an_entity_directory(self, make_inventory): - """A track table is refused by name rather than ignored.""" - files = { - "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", - "fiber_arrays/DAS.L001/coupling.csv": "start_distance\n0\n", - } - with pytest.raises(InvalidInventoryError, match="track table"): - make_inventory(files) - - def test_optical_path_epoch_directory(self, make_inventory): - """An optical path epoch is refused by name rather than ignored.""" - files = { - "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", - "fiber_arrays/DAS.L001/path@2024-05-12T103000/attrs.yaml": ( - "object_type: OpticalPath\n" - ), - } - with pytest.raises(InvalidInventoryError, match="optical path epoch"): - make_inventory(files) + """A table lives where the entity whose tracks it holds does.""" def test_track_table_outside_an_entity_directory(self, make_inventory): """A table lives beside the attrs file of the entity it describes.""" @@ -1060,3 +1057,972 @@ def test_inventory_models_refuse_unknown_input(self): model(**fields, nonsense_key=1) checked.append(model) assert len(checked) == 9 + + +# A fiber array whose optical path states every track shape at once. The +# path is 1000.1 m long, so every interval below sits inside it. +TRACKS = { + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\nname: array\n", + "fiber_arrays/DAS.L001/path/attrs.yaml": "object_type: OpticalPath\nname: main\n", + "fiber_arrays/DAS.L001/path/optical_components.csv": ( + "sequence,object_type,optical_length,name,fiber_number,fiber_color\n" + "2,Splice,0.1,splice 1,,\n" + "1,FiberSegment,1000.0,fiber 1,1,blue\n" + ), + "fiber_arrays/DAS.L001/path/coupling.csv": ( + "start_distance,end_distance,coupling_type,description\n" + "0,340,conduit,\n" + "340,355,trench,backfilled\n" + ), + "fiber_arrays/DAS.L001/path/annotations.csv": ( + "start_distance,end_distance,group,value\n" + "0,340,rock_type,granite\n" + "0,120,noisy,true\n" + "120,340,frost_depth,1.2\n" + ), + "fiber_arrays/DAS.L001/path/geometry.csv": ( + "segment,distance,longitude,latitude,elevation\n" + "S100,100.0,-117.0,40.0,687.0\n" + "S100,102.0,-117.1,40.1,685.0\n" + ), +} + + +def one_path(inventory): + """Return the single optical path of a loaded inventory.""" + return inventory.networks[0].fiber_arrays[0].optical_paths[0] + + +class TestTrackTables: + """Tests for the CSV half of the format.""" + + def test_every_shape_at_once(self, make_inventory): + """One path states all four tables, and each arrives whole.""" + path = one_path(make_inventory({**MINIMAL, **TRACKS})) + assert [x.name for x in path.optical_components] == ["fiber 1", "splice 1"] + assert [x.coupling_type for x in path.coupling] == ["conduit", "trench"] + assert {x.group for x in path.annotations} == { + "rock_type", + "noisy", + "frost_depth", + } + assert [x.name for x in path.geometry] == ["S100"] + + def test_rows_are_read_in_the_order_they_state(self, make_inventory): + """Sequence decides the order, never row position.""" + path = one_path(make_inventory({**MINIMAL, **TRACKS})) + # The file lists the splice first; sequence puts it second. + assert [x.object_type for x in path.optical_components] == [ + "FiberSegment", + "Splice", + ] + + @pytest.mark.parametrize( + ("text", "expected"), + [ + ("granite", "granite"), + ("true", True), + ("false", False), + ("1.2", 1.2), + ("2", 2), + # Integral, and how a spreadsheet writes a large one. Read from + # the text this raises; read from the number it does not. + ("1e3", 1000), + ], + ) + def test_a_value_is_read_as_its_text_states(self, make_inventory, text, expected): + """A CSV has no types, so an annotation's value is read by content.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/annotations.csv": ( + f"start_distance,end_distance,group,value\n0,340,g,{text}\n" + ), + } + value = one_path(make_inventory(files)).annotations[0].value + assert value == expected and isinstance(value, type(expected)) + + def test_a_group_holding_two_kinds(self, make_inventory): + """A group's kind decides its shape, so one group holds one kind.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/annotations.csv": ( + "start_distance,end_distance,group,value\n" + "0,120,zone,north\n" + "120,340,zone,true\n" + ), + } + with pytest.raises(InvalidInventoryError, match="one group holds one kind"): + make_inventory(files) + + def test_an_empty_cell_is_unset(self, make_inventory): + """An empty cell means unset, never an empty string.""" + path = one_path(make_inventory({**MINIMAL, **TRACKS})) + splice = path.optical_components[1] + # The splice row leaves the fiber columns empty; they are a + # FiberSegment's fields, and it is not one. + assert splice.description == "" + assert path.coupling[0].description == "" + assert path.coupling[1].description == "backfilled" + + def test_a_geometry_names_its_axes_the_way_its_frame_does(self, make_inventory): + """Coordinates are stored canonically, whatever the CRS calls them.""" + path = one_path(make_inventory({**MINIMAL, **TRACKS})) + geometry = path.geometry[0] + assert geometry.distance == (100.0, 102.0) + # longitude, latitude, elevation are axes 0, 1, 2 of the default CRS. + assert geometry.coordinates[0] == (-117.0, 40.0, 687.0) + + def test_a_declared_frame_renames_the_axes(self, make_inventory): + """The envelope decides which headers a geometry table may state.""" + files = { + **MINIMAL, + **TRACKS, + "inventory.yaml": ( + "object_type: Inventory\n" + "coordinate_reference_system:\n" + " authority: EPSG\n" + " code: '32611'\n" + " coordinate_labels: [x, y, elevation]\n" + " units: [meter, meter, meter]\n" + ), + "fiber_arrays/DAS.L001/path/geometry.csv": ( + "segment,distance,x,y,elevation\n" + "S100,100.0,2562048.25,1137365.53,687.0\n" + "S100,102.0,2562048.17,1137365.63,685.0\n" + ), + } + geometry = one_path(make_inventory(files)).geometry[0] + assert geometry.coordinates[0] == (2562048.25, 1137365.53, 687.0) + + def test_a_header_the_frame_does_not_declare(self, make_inventory): + """A geometry header disagreeing with the frame raises.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/geometry.csv": ( + "segment,distance,x,y,z\nS100,100.0,1.0,2.0,3.0\n" + ), + } + with pytest.raises(InvalidInventoryError, match="its frame declares"): + make_inventory(files) + + def test_a_point_missing_an_axis(self, make_inventory): + """A coordinate states every axis its frame declares.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/geometry.csv": ( + "segment,distance,longitude,latitude,elevation\n" + "S100,100.0,-117.0,40.0,687.0\n" + "S100,102.0,-117.1,,685.0\n" + ), + } + with pytest.raises(InvalidInventoryError, match="leaves latitude empty"): + make_inventory(files) + + def test_a_single_object_table(self, make_inventory): + """A map has no grouping column: every point belongs to the one map.""" + files = { + "acquisitions/DAS.L001.02.DEC/attrs.yaml": ( + "object_type: Acquisition\nspatial_interval: 1.0\n" + ), + "acquisitions/DAS.L001.02.DEC/distance_map.csv": ( + "channel,distance\n512,500.0\n1710,1698.0\n" + ), + } + out = make_inventory(files) + acquisition = out.networks[0].fiber_arrays[0].acquisitions[0] + assert acquisition.distance_map.channel == (512.0, 1710.0) + assert acquisition.channel_to_distance([512])[0] == 500.0 + + def test_a_stem_naming_no_attribute(self, make_inventory): + """A table is matched to the model by name, so a typo is a typo.""" + files = { + **MINIMAL, + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/geometrys.csv": "segment,distance\nS,0\n", + } + with pytest.raises(InvalidInventoryError, match="names no attribute"): + make_inventory(files) + + def test_a_stem_naming_a_field_which_is_not_row_shaped(self, make_inventory): + """Only an attribute rows can build may be stated as a table.""" + files = { + **MINIMAL, + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/name.csv": "a,b\n1,2\n", + } + with pytest.raises(InvalidInventoryError, match="does not read as a table"): + make_inventory(files) + + def test_a_row_shaped_attribute_with_no_table(self, make_inventory): + """Station.channels is row-shaped and still has no table form. + + The message must not tell the author otherwise, which is what it + did while it said "not a row-shaped attribute". + """ + files = { + "stations/DAS.STA1/attrs.yaml": "object_type: Station\n", + "stations/DAS.STA1/channels.csv": "code,location_code\nHHZ,00\n", + } + assert "channels" in inv.Station.model_fields + with pytest.raises(InvalidInventoryError, match="does not read as a table"): + make_inventory(files) + + def test_stated_inline_and_as_a_table(self, make_inventory): + """One fact is spelled once.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/attrs.yaml": ( + "object_type: OpticalPath\nname: main\n" + "coupling:\n" + " - start_distance: 0.0\n" + " end_distance: 10.0\n" + " coupling_type: trench\n" + ), + } + with pytest.raises(InvalidInventoryError, match="spelled once"): + make_inventory(files) + + def test_a_table_missing_the_column_it_is_read_by(self, make_inventory): + """Components are placed by sequence, so a table without one raises.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/optical_components.csv": ( + "object_type,optical_length,name\nFiberSegment,1000.0,fiber 1\n" + ), + } + with pytest.raises(InvalidInventoryError, match="no sequence column"): + make_inventory(files) + + def test_a_column_stated_twice(self, make_inventory): + """A repeated header means two columns claim one field.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/coupling.csv": ( + "start_distance,end_distance,coupling_type,coupling_type\n" + "0,340,conduit,trench\n" + ), + } + with pytest.raises(InvalidInventoryError, match="more than once"): + make_inventory(files) + + +class TestPathEpochs: + """`path` is the one reserved container stem, and it holds a lineage.""" + + def test_a_bare_path_is_the_first_epoch(self, make_inventory): + """The bare directory is the blank location, starting where the array does.""" + path = one_path(make_inventory({**MINIMAL, **TRACKS})) + assert path.location_code == "" + assert pd.isnull(path.start_time) + + def test_a_location_joins_the_stem_with_a_dot(self, make_inventory): + """Each location code is its own lineage.""" + files = { + **MINIMAL, + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/path.00/attrs.yaml": ( + "object_type: OpticalPath\nname: first\n" + ), + "fiber_arrays/DAS.L001/path.01/attrs.yaml": ( + "object_type: OpticalPath\nname: second\n" + ), + } + paths = make_inventory(files).networks[0].fiber_arrays[0].optical_paths + assert {x.location_code: x.name for x in paths} == { + "00": "first", + "01": "second", + } + + def test_an_epoch_ends_where_the_next_begins(self, make_inventory): + """Sorted by start, each epoch runs until its successor.""" + files = { + **MINIMAL, + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/path/attrs.yaml": ( + "object_type: OpticalPath\nname: first\n" + ), + "fiber_arrays/DAS.L001/path@2024-05-12T103000/attrs.yaml": ( + "object_type: OpticalPath\nname: second\n" + ), + } + paths = { + x.name: x + for x in make_inventory(files).networks[0].fiber_arrays[0].optical_paths + } + boundary = np.datetime64("2024-05-12T10:30:00", "ns") + # The first epoch was left open and the directory closed it. + assert paths["first"].end_time == boundary + assert paths["second"].start_time == boundary + assert pd.isnull(paths["second"].end_time) + + def test_an_epoch_may_end_early(self, make_inventory): + """A dark interval: an epoch states an end before its successor.""" + files = { + **MINIMAL, + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/path/attrs.yaml": ( + "object_type: OpticalPath\nname: first\nend_time: 2024-01-01\n" + ), + "fiber_arrays/DAS.L001/path@2024-05-12T103000/attrs.yaml": ( + "object_type: OpticalPath\nname: second\n" + ), + } + paths = { + x.name: x + for x in make_inventory(files).networks[0].fiber_arrays[0].optical_paths + } + assert paths["first"].end_time == np.datetime64("2024-01-01", "ns") + + def test_an_epoch_may_not_end_late(self, make_inventory): + """An epoch cannot claim time its successor already holds.""" + files = { + **MINIMAL, + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/path/attrs.yaml": ( + "object_type: OpticalPath\nname: first\nend_time: 2025-01-01\n" + ), + "fiber_arrays/DAS.L001/path@2024-05-12T103000/attrs.yaml": ( + "object_type: OpticalPath\nname: second\n" + ), + } + with pytest.raises( + InvalidInventoryError, match="after the epoch which follows" + ): + make_inventory(files) + + def test_two_names_for_one_epoch(self, make_inventory): + """Epoch-name uniqueness is temporal rather than textual.""" + files = { + **MINIMAL, + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/path@2024-06-01/attrs.yaml": ( + "object_type: OpticalPath\nname: first\n" + ), + "fiber_arrays/DAS.L001/path@2024-06-01T000000/attrs.yaml": ( + "object_type: OpticalPath\nname: second\n" + ), + } + with pytest.raises(InvalidInventoryError, match="two spellings of one epoch"): + make_inventory(files) + + def test_a_restated_location_must_agree(self, make_inventory): + """The directory name is an address like any other.""" + files = { + **MINIMAL, + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/path.00/attrs.yaml": ( + "object_type: OpticalPath\nlocation_code: '01'\n" + ), + } + with pytest.raises(InvalidInventoryError, match="must agree with the name"): + make_inventory(files) + + def test_a_path_directory_declaring_something_else(self, make_inventory): + """The declared type must agree with the container.""" + files = { + **MINIMAL, + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/path/attrs.yaml": "object_type: Acquisition\n", + } + with pytest.raises(InvalidInventoryError, match="holds an OpticalPath"): + make_inventory(files) + + def test_a_path_where_nothing_holds_one(self, make_inventory): + """An acquisition has no optical paths, so it cannot hold the epoch.""" + files = { + "acquisitions/DAS.L001..RAW/attrs.yaml": "object_type: Acquisition\n", + "acquisitions/DAS.L001..RAW/path/attrs.yaml": "object_type: OpticalPath\n", + } + with pytest.raises(InvalidInventoryError, match="which holds none"): + make_inventory(files) + + def test_paths_stated_inline_and_as_directories(self, make_inventory): + """One fact is spelled once.""" + files = { + **MINIMAL, + "fiber_arrays/DAS.L001/attrs.yaml": ( + "object_type: FiberArray\noptical_paths:\n - name: inline\n" + ), + "fiber_arrays/DAS.L001/path/attrs.yaml": ( + "object_type: OpticalPath\nname: from a directory\n" + ), + } + with pytest.raises(InvalidInventoryError, match="spelled once"): + make_inventory(files) + + def test_an_object_misfiled_inside_a_path(self, make_inventory): + """A path epoch holds its own attrs and tracks, like any entity.""" + files = { + **MINIMAL, + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/path/attrs.yaml": "object_type: OpticalPath\n", + "fiber_arrays/DAS.L001/path/DAS.L002.yaml": "object_type: FiberArray\n", + } + with pytest.raises(InvalidInventoryError, match="holds only its attrs"): + make_inventory(files) + + def test_a_path_resolves_through_the_inventory(self, make_inventory): + """A directory-built path is reached the way any other is.""" + out = make_inventory({**MINIMAL, **TRACKS}) + resolved = out.resolve("DAS.L001..RAW") + assert resolved.optical_path.name == "main" + assert len(resolved.optical_path.coupling) == 2 + + +class TestUnreadableTables: + """A table which claims a place in the format must be readable.""" + + def test_a_table_which_is_not_text(self, tmp_path): + """A file which cannot be read at all says which one it was.""" + root = write_inventory(tmp_path / "binary", {**MINIMAL, **TRACKS}) + (root / "fiber_arrays/DAS.L001/path/coupling.csv").write_bytes(b"\xff\xfe\x00") + with pytest.raises(InvalidInventoryError, match="Could not read"): + dc.inventory(root) + + def test_a_table_with_no_columns(self, make_inventory): + """An empty file states no track.""" + files = {**MINIMAL, **TRACKS, "fiber_arrays/DAS.L001/path/coupling.csv": ""} + with pytest.raises(InvalidInventoryError, match="no columns"): + make_inventory(files) + + def test_a_non_numeric_order_column(self, make_inventory): + """Rows are placed by their order column, so it must be a number.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/optical_components.csv": ( + "sequence,object_type,optical_length,name\n" + "first,FiberSegment,1000.0,fiber 1\n" + ), + } + with pytest.raises(InvalidInventoryError, match="non-numeric sequence"): + make_inventory(files) + + def test_a_column_no_row_states(self, make_inventory): + """A column every row leaves empty states nothing, and is not empty.""" + files = { + "acquisitions/DAS.L001.02.DEC/attrs.yaml": ( + "object_type: Acquisition\nspatial_interval: 1.0\n" + ), + "acquisitions/DAS.L001.02.DEC/distance_map.csv": ( + "channel,instrument_distance,distance\n512,,500.0\n1710,,1698.0\n" + ), + } + acquisition = make_inventory(files).networks[0].fiber_arrays[0].acquisitions[0] + # Unset, rather than a tuple of nothing, which the model would refuse. + assert acquisition.distance_map.instrument_distance is None + + def test_an_annotation_stating_no_value(self, make_inventory): + """A membership group's value is its default, not a parsed cell.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/annotations.csv": ( + "start_distance,end_distance,group,value\n0,120,noisy,\n" + ), + } + annotation = one_path(make_inventory(files)).annotations[0] + assert annotation.value is True + + def test_a_path_restating_a_start_which_disagrees(self, make_inventory): + """A path directory's name is a restated address like any other.""" + files = { + **MINIMAL, + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/path@2024-06-01/attrs.yaml": ( + "object_type: OpticalPath\nstart_time: 2024-06-02\n" + ), + } + with pytest.raises(InvalidInventoryError, match="must agree with the name"): + make_inventory(files) + + def test_an_envelope_declaring_an_unreadable_frame(self, make_inventory): + """The frame builds once, with the envelope which states it. + + Which is why the loader rebuilds it without a guard of its own: a + second check there could only report what this one already has. + """ + files = { + **MINIMAL, + "inventory.yaml": ( + "object_type: Inventory\n" + "coordinate_reference_system:\n" + " coordinate_labels: [x, x, z]\n" + ), + } + with pytest.raises(InvalidInventoryError, match="Could not read the envelope"): + make_inventory(files) + + @pytest.mark.parametrize("row", ["0,340,conduit,extra", "0,340"]) + def test_a_row_which_is_not_its_header_wide(self, make_inventory, row): + """A row states one cell per column or it is not a row. + + Pandas refuses neither: a surplus cell silently pushes the first + column into the index, so `0,340,conduit,extra` would load as + start_distance=340, end_distance=conduit, coupling_type=extra -- + every value one field left of what it says. + """ + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/coupling.csv": ( + f"start_distance,end_distance,coupling_type\n{row}\n" + ), + } + with pytest.raises(InvalidInventoryError, match="its header names 3 columns"): + make_inventory(files) + + def test_a_sequence_which_places_two_rows_alike(self, make_inventory): + """Components tile the path, so no two may claim one place.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/optical_components.csv": ( + "sequence,object_type,optical_length,name\n" + "1,FiberSegment,100.0,a\n" + "1,Splice,0.1,b\n" + ), + } + with pytest.raises(InvalidInventoryError, match="does not say which row"): + make_inventory(files) + + def test_a_cell_which_does_not_pertain_to_its_row(self, make_inventory): + """The model refuses a field its own type does not declare.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/optical_components.csv": ( + "sequence,object_type,optical_length,name,fiber_color\n" + "1,Splice,0.1,b,blue\n" + ), + } + # A splice has no fiber colour; only a segment does. + with pytest.raises(InvalidInventoryError, match="Could not read OpticalPath"): + make_inventory(files) + + +# Which model declares each table's attribute. Spelled out rather than +# searched for, so a table moving between models is a change here. +DECLARED_BY = { + "optical_components": inv.OpticalPath, + "coupling": inv.OpticalPath, + "annotations": inv.OpticalPath, + "geometry": inv.OpticalPath, + "distance_map": inv.Acquisition, +} + + +class TestTableRegistry: + """The table registry must stay pinned to the models.""" + + def test_every_stem_is_a_field_of_its_model(self): + """A stem which names no field could never be authored.""" + assert set(loader._TABLES) == set(DECLARED_BY) + for stem, model in DECLARED_BY.items(): + assert stem in model.model_fields + + def test_a_grouping_or_ordering_column_is_not_a_stem(self): + """The columns a table is read by are structural, not attributes.""" + for stem, table in loader._TABLES.items(): + for column in (table.group, table.order): + assert column is None or column not in loader._TABLES + assert stem # every key names something + + def test_only_a_placing_table_drops_its_order_column(self): + """A column dropped where it is not scaffolding would vanish unseen.""" + placing = {k for k, v in loader._TABLES.items() if v.places} + assert placing == {"optical_components"} + # And that column is the one the model has no field for. + assert "sequence" not in inv.OpticalPath.model_fields + + +class TestSilentlyLostRows: + """A row which states a place must keep it, or say it has none.""" + + def test_a_blank_grouping_cell(self, make_inventory): + """A null grouping key drops its row from every group, unasked.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/geometry.csv": ( + "segment,distance,longitude,latitude,elevation\n" + "S100,100.0,-117.0,40.0,687.0\n" + ",102.0,-117.1,40.1,685.0\n" + ), + } + with pytest.raises(InvalidInventoryError, match="state no place"): + make_inventory(files) + + def test_every_row_blank_in_the_grouping_column(self, make_inventory): + """The whole track would otherwise vanish and the path load empty.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/geometry.csv": ( + "segment,distance,longitude,latitude,elevation\n" + ",100.0,-117.0,40.0,687.0\n" + ",102.0,-117.1,40.1,685.0\n" + ), + } + with pytest.raises(InvalidInventoryError, match="state no place"): + make_inventory(files) + + def test_a_single_blank_ordering_cell(self, make_inventory): + """One blank is as unplaced as two, though only two ever collide.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/optical_components.csv": ( + "sequence,object_type,optical_length,name\n" + "1,FiberSegment,100.0,a\n" + ",Splice,0.1,b\n" + ), + } + with pytest.raises(InvalidInventoryError, match="state no place"): + make_inventory(files) + + def test_a_column_some_points_state_and_others_do_not(self, make_inventory): + """Columns become parallel arrays, so a gap would pair the unpaired.""" + files = { + "acquisitions/DAS.L001.02.DEC/attrs.yaml": ( + "object_type: Acquisition\nspatial_interval: 1.0\n" + ), + # Channel 5 states no distance and 100 states no channel; read + # column by column they would pair up as though they had. + "acquisitions/DAS.L001.02.DEC/distance_map.csv": ( + "channel,instrument_distance,distance\n5,,10.0\n,20.0,100.0\n" + ), + } + with pytest.raises(InvalidInventoryError, match="stated by every point"): + make_inventory(files) + + def test_a_stray_ordering_column_is_refused(self, make_inventory): + """A sequence column is scaffolding only where the table says so.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/coupling.csv": ( + "sequence,start_distance,end_distance,coupling_type\n1,0,340,conduit\n" + ), + } + # Coupling is not placed by sequence, so the model calls it unknown. + with pytest.raises(InvalidInventoryError, match="Could not read OpticalPath"): + make_inventory(files) + + +class TestPathEpochsBelongToTheirArray: + """An epoch cannot start before the entity which holds it.""" + + def test_a_bare_path_starts_where_its_array_does(self, make_inventory): + """Left unset it would claim the unbounded past, before the array.""" + # No acquisition: an undated one could not live in a dated array, + # which is (a)'s containment rule and not what this pins. + files = { + "fiber_arrays/DAS.L001@2024-01-01/attrs.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001@2024-01-01/path/attrs.yaml": ( + "object_type: OpticalPath\nname: main\n" + ), + } + array = make_inventory(files).networks[0].fiber_arrays[0] + assert array.start_time == np.datetime64("2024-01-01", "ns") + assert array.optical_paths[0].start_time == array.start_time + + def test_an_undated_array_leaves_its_path_undated(self, make_inventory): + """There is nothing to inherit, so the path keeps its own unset start.""" + path = one_path(make_inventory({**MINIMAL, **TRACKS})) + assert pd.isnull(path.start_time) + + @pytest.mark.skipif( + not KEEPS_TRAILING_DOT, reason="this filesystem holds one of the two" + ) + def test_two_undated_epochs_of_one_lineage(self, make_inventory): + """NaT equals nothing, so this collision has to be found by hand.""" + files = { + **MINIMAL, + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/path/attrs.yaml": ( + "object_type: OpticalPath\nname: first\n" + ), + # `path.` is the same blank location as `path`, and neither + # names an instant. + "fiber_arrays/DAS.L001/path./attrs.yaml": ( + "object_type: OpticalPath\nname: second\n" + ), + } + with pytest.raises(InvalidInventoryError, match="two spellings of one epoch"): + make_inventory(files) + + def test_a_lineage_error_names_a_file_which_exists(self, tmp_path): + """An error which names a file nobody can open helps nobody.""" + root = write_inventory( + tmp_path / "late_end", + { + **MINIMAL, + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/path/attrs.yaml": ( + "object_type: OpticalPath\nend_time: 2025-01-01\n" + ), + "fiber_arrays/DAS.L001/path@2024-05-12T103000/attrs.yaml": ( + "object_type: OpticalPath\n" + ), + }, + ) + with pytest.raises(InvalidInventoryError) as info: + dc.inventory(root) + named = [x for x in str(info.value).split() if x.endswith(".yaml")] + assert named, str(info.value) + for name in named: + assert (root / "fiber_arrays/DAS.L001" / name).exists() + + +class TestGapsMutationTestingFound: + """Cases the suite asserted around rather than through.""" + + def test_points_gather_into_the_segment_which_names_them(self, make_inventory): + """Every point table elsewhere holds one object, so grouping is free. + + A loader ignoring the grouping column entirely would pass those, + merging every point into a single segment. + """ + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/geometry.csv": ( + "segment,distance,longitude,latitude,elevation\n" + "S100,100.0,-117.0,40.0,687.0\n" + "S120,200.0,-117.2,40.2,690.0\n" + "S100,102.0,-117.1,40.1,685.0\n" + "S120,202.0,-117.3,40.3,688.0\n" + ), + } + geometry = {x.name: x for x in one_path(make_inventory(files)).geometry} + assert set(geometry) == {"S100", "S120"} + # Interleaved in the file; gathered by name and ordered by distance. + assert geometry["S100"].distance == (100.0, 102.0) + assert geometry["S120"].distance == (200.0, 202.0) + assert geometry["S100"].coordinates[1] == (-117.1, 40.1, 685.0) + + def test_a_wrong_cell_names_the_field_it_could_not_take(self, make_inventory): + """Matching only 'Could not read' would pass for any path failure.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/optical_components.csv": ( + "sequence,object_type,optical_length,name,fiber_color\n" + "1,Splice,0.1,b,blue\n" + ), + } + with pytest.raises(InvalidInventoryError, match="fiber_color"): + make_inventory(files) + + @pytest.mark.parametrize("order", [("1", "1.5"), ("1.5", "1")]) + def test_an_int_and_a_float_are_one_kind(self, make_inventory, order): + """The model reads them alike, so row order cannot decide this.""" + first, second = order + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/annotations.csv": ( + f"start_distance,end_distance,group,value\n" + f"0,120,thickness,{first}\n120,340,thickness,{second}\n" + ), + } + values = [x.value for x in one_path(make_inventory(files)).annotations] + assert sorted(values) == [1, 1.5] + + @pytest.mark.parametrize("text", ["TRUE", "True", " true "]) + def test_a_boolean_however_a_spreadsheet_writes_it(self, make_inventory, text): + """Excel writes TRUE; the cell is stripped and folded before reading.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/annotations.csv": ( + f"start_distance,end_distance,group,value\n0,120,noisy,{text}\n" + ), + } + assert one_path(make_inventory(files)).annotations[0].value is True + + def test_a_decimal_point_keeps_a_value_a_float(self, make_inventory): + """1.0 is written as a float and stays one, unlike 1.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/annotations.csv": ( + "start_distance,end_distance,group,value\n0,120,thickness,1.0\n" + ), + } + value = one_path(make_inventory(files)).annotations[0].value + assert isinstance(value, float) and value == 1.0 + + def test_an_epoch_ending_exactly_where_the_next_begins(self, make_inventory): + """The ordinary explicit close, which must not read as overlapping.""" + files = { + **MINIMAL, + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/path/attrs.yaml": ( + "object_type: OpticalPath\nname: first\nend_time: 2024-05-12T10:30:00\n" + ), + "fiber_arrays/DAS.L001/path@2024-05-12T103000/attrs.yaml": ( + "object_type: OpticalPath\nname: second\n" + ), + } + paths = { + x.name: x + for x in make_inventory(files).networks[0].fiber_arrays[0].optical_paths + } + assert paths["first"].end_time == paths["second"].start_time + + def test_lineages_of_two_locations_stay_apart(self, make_inventory): + """Each location is its own lineage, so neither closes the other.""" + files = { + **MINIMAL, + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/path.00/attrs.yaml": ( + "object_type: OpticalPath\nname: a\n" + ), + "fiber_arrays/DAS.L001/path.01@2024-06-01/attrs.yaml": ( + "object_type: OpticalPath\nname: b\n" + ), + } + paths = { + x.name: x + for x in make_inventory(files).networks[0].fiber_arrays[0].optical_paths + } + # Pooled into one lineage, `a` would have been closed by `b`. + assert pd.isnull(paths["a"].end_time) + assert pd.isnull(paths["b"].end_time) + + def test_a_shouted_table_suffix_and_a_hidden_sidecar(self, tmp_path): + """A table is found however its suffix is cased, and a dotfile is not.""" + root = write_inventory(tmp_path / "sidecars", {**MINIMAL, **TRACKS}) + entity = root / "fiber_arrays/DAS.L001/path" + (entity / "coupling.csv").rename(entity / "coupling.CSV") + (entity / "._geometry.csv").write_text("junk that is not a table\n") + path = one_path(dc.inventory(root)) + assert len(path.coupling) == 2 + assert [x.name for x in path.geometry] == ["S100"] + + def test_a_blank_line_in_a_table_body(self, make_inventory): + """A trailing or stray blank line states no row rather than a short one.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/coupling.csv": ( + "start_distance,end_distance,coupling_type\n0,340,conduit\n\n" + ), + } + assert len(one_path(make_inventory(files)).coupling) == 1 + + def test_a_cell_holding_the_word_na(self, make_inventory): + """Only an empty cell is unset; NA is what the author wrote.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/coupling.csv": ( + "start_distance,end_distance,coupling_type,description\n" + "0,340,conduit,NA\n" + ), + } + assert one_path(make_inventory(files)).coupling[0].description == "NA" + + def test_two_undated_epochs_collide_on_every_filesystem(self, tmp_path): + """The rule above, where the directories which state it cannot exist. + + Windows keeps no trailing dot, so `path.` and `path` are one + directory there; the comparison is pinned here instead. + """ + first = inv.OpticalPath(name="first") + second = inv.OpticalPath(name="second") + sources = {id(first): tmp_path / "a.yaml", id(second): tmp_path / "b.yaml"} + assert pd.isnull(first.start_time) and pd.isnull(second.start_time) + with pytest.raises(InvalidInventoryError, match="two spellings of one epoch"): + loader._close_lineages([first, second], sources) + + +class TestPRReviewFindings: + """Regressions for what the review bots found on the pull request.""" + + def test_a_gap_names_the_line_it_is_on(self, make_inventory): + """The frame is sorted and split by then, so positions lie.""" + files = { + "acquisitions/DAS.L001.02.DEC/attrs.yaml": ( + "object_type: Acquisition\nspatial_interval: 1.0\n" + ), + # Sorted by distance these rows reverse, so a position within + # the sorted frame is not the line the author would open. + "acquisitions/DAS.L001.02.DEC/distance_map.csv": ( + "channel,instrument_distance,distance\n" + "1,10,300\n" # line 2 + "2,,100\n" # line 3, the gap + "3,30,200\n" # line 4 + ), + } + with pytest.raises(InvalidInventoryError, match=r"row\(s\) 3") as info: + make_inventory(files) + # Counting within the sorted frame would have said row 2. + assert "row(s) 2" not in str(info.value) + + def test_an_empty_single_object_table(self, make_inventory): + """A header and no rows describes no map, rather than raising bare.""" + files = { + "acquisitions/DAS.L001.02.DEC/attrs.yaml": ( + "object_type: Acquisition\nspatial_interval: 1.0\n" + ), + "acquisitions/DAS.L001.02.DEC/distance_map.csv": "channel,distance\n", + } + with pytest.raises(InvalidInventoryError, match="describes no distance_map"): + make_inventory(files) + + @pytest.mark.skipif(FOLDS_CASE, reason="this filesystem holds one of the two") + def test_two_path_directories_differing_only_by_case(self, make_inventory): + """Two lineages here become one wherever the inventory is copied.""" + files = { + **MINIMAL, + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", + "fiber_arrays/DAS.L001/path.aa/attrs.yaml": ( + "object_type: OpticalPath\nname: a\n" + ), + "fiber_arrays/DAS.L001/path.AA/attrs.yaml": ( + "object_type: OpticalPath\nname: b\n" + ), + } + with pytest.raises(InvalidInventoryError, match="differ only by case"): + make_inventory(files) + + def test_a_symlinked_path_directory(self, tmp_path): + """A symlink would read an epoch from outside the inventory.""" + outside = tmp_path / "elsewhere" / "path" + outside.mkdir(parents=True) + (outside / "attrs.yaml").write_text("object_type: OpticalPath\nname: away\n") + root = write_inventory( + tmp_path / "linked", + { + **MINIMAL, + "fiber_arrays/DAS.L001/attrs.yaml": "object_type: FiberArray\n", + }, + ) + os.symlink(outside, root / "fiber_arrays/DAS.L001/path") + # Not part of the format, so the array simply has no optical path. + assert not dc.inventory(root).networks[0].fiber_arrays[0].optical_paths + + def test_a_large_table_is_not_held_twice(self, make_inventory): + """The width check streams, so only the frame stays resident.""" + rows = "\n".join(f"S100,{x}.0,-117.0,40.0,687.0" for x in range(2000)) + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/optical_components.csv": ( + "sequence,object_type,optical_length,name\n1,FiberSegment,3000.0,a\n" + ), + "fiber_arrays/DAS.L001/path/geometry.csv": ( + f"segment,distance,longitude,latitude,elevation\n{rows}\n" + ), + } + assert len(one_path(make_inventory(files)).geometry[0].distance) == 2000