diff --git a/dascore/__init__.py b/dascore/__init__.py index b1e385ae5..ac6cb40a4 100644 --- a/dascore/__init__.py +++ b/dascore/__init__.py @@ -24,7 +24,11 @@ reset_config, set_config, ) -from dascore.examples import get_example_patch, get_example_spool +from dascore.examples import ( + get_example_inventory, + get_example_patch, + get_example_spool, +) from dascore.io.core import get_format, read, scan, scan_payloads, scan_to_df, write from dascore.units import get_quantity, get_unit from dascore.utils.patch import patch_function diff --git a/dascore/core/annotation_loader.py b/dascore/core/annotation_loader.py index f7e276863..ab9b75efe 100644 --- a/dascore/core/annotation_loader.py +++ b/dascore/core/annotation_loader.py @@ -40,6 +40,7 @@ import numpy as np import pandas as pd +import yaml from pydantic import ValidationError from dascore.core.annotations import ( @@ -60,7 +61,7 @@ ) from dascore.exceptions import InvalidAnnotationError, ParameterError from dascore.models.registry import TAG_FIELD -from dascore.utils.misc import iterate, optional_import +from dascore.utils.misc import iterate from dascore.utils.paths import quote_path from dascore.utils.tables import ( parse_cell, @@ -119,7 +120,6 @@ def _read_object(path: Path) -> dict[str, Any]: msg = f"Could not parse JSON from {quote_path(path)}: {error}." raise ParameterError(msg) from error else: - yaml = optional_import("yaml", required_for="YAML annotation storage") try: data = yaml.safe_load(text) except yaml.YAMLError as error: diff --git a/dascore/core/inventory.py b/dascore/core/inventory.py index f8fab696d..fd89f8df1 100644 --- a/dascore/core/inventory.py +++ b/dascore/core/inventory.py @@ -34,6 +34,7 @@ from uuid import uuid4 import numpy as np +import yaml from pydantic import ( AfterValidator, BeforeValidator, @@ -64,7 +65,6 @@ from dascore.utils.misc import ( check_code, is_strictly_monotonic, - optional_import, validate_acquisition_key, ) from dascore.utils.namespace import NamespaceOwner @@ -2501,7 +2501,6 @@ def from_yaml(cls, text: str) -> Self: "Load a path with dascore.inventory." ) raise InvalidInventoryError(msg) - yaml = optional_import("yaml", required_for="YAML inventory parsing") try: data = yaml.safe_load(text) except yaml.YAMLError as error: @@ -2554,13 +2553,10 @@ def inventory_to_yaml(inventory: Inventory, path: str | Path | None = None) -> s -------- >>> import dascore as dc >>> _, inventory = dc.examples.inventory_patch_pair() - >>> # Writing YAML needs pyyaml, which is not a core dependency. - >>> text = inventory.io.to_yaml() # doctest: +SKIP - >>> dc.inventory(text) == inventory # doctest: +SKIP + >>> text = inventory.io.to_yaml() + >>> dc.inventory(text) == inventory True """ - yaml = optional_import("yaml", required_for="YAML inventory serialization") - # Everything defaulted is dropped, so the document records which # envelope it was written against even when that is the default. dumped = inventory.model_dump(mode="json", exclude_defaults=True) diff --git a/dascore/core/inventory_loader.py b/dascore/core/inventory_loader.py index a105f5db8..202843038 100644 --- a/dascore/core/inventory_loader.py +++ b/dascore/core/inventory_loader.py @@ -41,6 +41,7 @@ from typing import Any, NamedTuple import pandas as pd +import yaml from dascore.core.inventory import ( Acquisition, @@ -58,14 +59,10 @@ _overlapping_epochs, _times_equal, ) -from dascore.exceptions import ( - InvalidInventoryError, - MissingOptionalDependencyError, - ParameterError, -) +from dascore.exceptions import InvalidInventoryError, ParameterError from dascore.models import InventoryModel, TimeRangedModel from dascore.models.registry import TAG_FIELD -from dascore.utils.misc import check_code, optional_import +from dascore.utils.misc import check_code from dascore.utils.paths import quote_path as _quote from dascore.utils.tables import ( ordered_rows, @@ -221,7 +218,6 @@ def _read_object(path: Path) -> dict[str, Any]: msg = f"Could not parse JSON from {_quote(path)}: {error}." raise InvalidInventoryError(msg) from error else: - yaml = optional_import("yaml", required_for="YAML inventory serialization") try: data = yaml.safe_load(text) except yaml.YAMLError as error: @@ -245,7 +241,7 @@ def _declared_type(path: Path) -> str | None: """ try: data = _read_object(path) - except (InvalidInventoryError, MissingOptionalDependencyError): + except InvalidInventoryError: return None declared = data.get(TAG_FIELD) return declared if isinstance(declared, str) else None diff --git a/dascore/examples.py b/dascore/examples.py index e1936a6a7..cf9d303a2 100644 --- a/dascore/examples.py +++ b/dascore/examples.py @@ -2,6 +2,7 @@ from __future__ import annotations +import io import tempfile from collections.abc import Sequence from contextlib import suppress @@ -38,6 +39,7 @@ EXAMPLE_PATCHES = {} EXAMPLE_SPOOLS = {} +EXAMPLE_INVENTORIES = {} def _load_example_patch_from_file(path: str | Path) -> dc.Patch: @@ -845,3 +847,411 @@ def inventory_patch_pair(): ) ).check() return patch, inventory + + +@register_func(EXAMPLE_INVENTORIES, key="random_das") +def random_das_inventory() -> Inventory: + """A single-path inventory which resolves the random_das example patch.""" + return inventory_patch_pair()[1] + + +def get_example_inventory(example_name="random_das", **kwargs) -> Inventory: + """ + Load an example Inventory. + + Supported example inventories are: + ```{python} + #| echo: false + #| output: asis + from dascore.examples import EXAMPLE_INVENTORIES + + from dascore.utils.docs import objs_to_doc_df + + df = objs_to_doc_df(EXAMPLE_INVENTORIES) + print(df.to_markdown(index=False, stralign="center")) + ``` + + Parameters + ---------- + example_name + The name of the example to load. Options are listed above. + **kwargs + Passed to the corresponding functions to generate the inventory. + + Raises + ------ + (`UnknownExampleError`)['dascore.examples.UnknownExampleError`] if an + unregistered inventory is requested. + + Examples + -------- + >>> import dascore as dc + >>> inventory = dc.get_example_inventory("tunnel") + >>> len(inventory.networks) + 1 + """ + if example_name not in EXAMPLE_INVENTORIES: + msg = ( + f"No example inventory registered with name {example_name} " + f"Registered example inventories are {list(EXAMPLE_INVENTORIES)}" + ) + raise UnknownExampleError(msg) + return EXAMPLE_INVENTORIES[example_name](**kwargs) + + +# --- The tunnel inventory ------------------------------------------------- +# +# This builds the deployment the tunnel recipe walks through, and is the +# single definition of it: the recipe displays these very files rather +# than composing its own, so the page and the example cannot drift apart. + +_TUNNEL_RESOURCES = { + "telemetry-cable": ( + "object_type: Cable\n" + "name: tunnel telemetry cable\n" + "manufacturer: Corning\n" + "model: MIC tight-buffered 4F OS2\n" + "fiber_count: 4\n" + "description: The run in from the instrument room.\n" + ), + "connecting-cable": ( + "object_type: Cable\n" + "name: tunnel connecting cable\n" + "manufacturer: Corning\n" + "model: MIC tight-buffered 4F OS2\n" + "fiber_count: 4\n" + "description: The links between boxes, couplers, and borehole heads.\n" + ), + "borehole-cable": ( + "object_type: Cable\n" + "name: borehole sensing cable\n" + "manufacturer: Nerve Sensors\n" + "model: Epsilon\n" + "fiber_count: 4\n" + "description: Rock-coupled downhole cable with an armored pigtail.\n" + ), + "trench-cable": ( + "object_type: Cable\n" + "name: helically wound trench cable\n" + "manufacturer: Silixa\n" + "model: HWC\n" + "fiber_count: 1\n" + ), + "das-interrogator": ( + "object_type: Interrogator\n" + "name: tunnel DAS interrogator\n" + "manufacturer: Sintela\n" + "model: Onyxia\n" + "instrument_type: DAS interrogator\n" + ), + "repair-cord": ( + "object_type: Cable\nname: trench repair patch cord\nfiber_count: 1\n" + ), + "repair-box": ( + "object_type: Enclosure\nname: trench repair box\nenclosure_type: box\n" + ), +} + +# One enclosure per housing, because a resource_id names an asset rather than +# a kind: box A and box E are two boxes, and each borehole has its own +# turnaround down the hole. +for _label, _name in [ + ("splice-box-a", "splice box at A"), + ("splice-box-e", "splice box at E"), + ("turnaround-1", "borehole 1 turnaround housing"), + ("turnaround-2", "borehole 2 turnaround housing"), + ("turnaround-3", "borehole 3 turnaround housing"), +]: + _kind = "box" if "splice" in _label else "housing" + _TUNNEL_RESOURCES[_label] = ( + f"object_type: Enclosure\nname: tunnel {_name}\nenclosure_type: {_kind}\n" + ) + +_TUNNEL_COMPONENTS = """\ +sequence,object_type,optical_length,name,container +1,FiberSegment,1500.0,telemetry lead-in,telemetry-cable +2,Splice,0.0,splice at box A,splice-box-a +3,FiberSegment,2.5,drop into the trench,trench-cable +4,FiberSegment,25.0,trench B to the coil,trench-cable +5,FiberSegment,10.0,cable coil at C,trench-cable +6,FiberSegment,25.0,trench from the coil to D,trench-cable +7,FiberSegment,2.5,rise out of the trench,trench-cable +8,Splice,0.0,splice at box E,splice-box-e +9,FiberSegment,15.0,link E to borehole 3,connecting-cable +10,FiberSegment,20.0,borehole 3 down,borehole-cable +11,Splice,0.0,borehole 3 turnaround,turnaround-3 +12,FiberSegment,20.0,borehole 3 up,borehole-cable +13,FiberSegment,15.0,link borehole 3 to coupler G,connecting-cable +14,Connector,0.0,coupler G, +15,FiberSegment,15.0,link coupler G to borehole 2,connecting-cable +16,FiberSegment,20.0,borehole 2 down,borehole-cable +17,Splice,0.0,borehole 2 turnaround,turnaround-2 +18,FiberSegment,20.0,borehole 2 up,borehole-cable +19,FiberSegment,15.0,link borehole 2 to coupler H,connecting-cable +20,Connector,0.0,coupler H, +21,FiberSegment,15.0,link coupler H to borehole 1,connecting-cable +22,FiberSegment,20.0,borehole 1 down,borehole-cable +23,Splice,0.0,borehole 1 turnaround,turnaround-1 +24,FiberSegment,20.0,borehole 1 up,borehole-cable +25,FiberSegment,15.0,link borehole 1 back to box A,connecting-cable +26,Terminator,0.0,path end, +""" + +# The surveyed waypoints, lettered as the recipe's drawing letters them. +_TUNNEL_A = (100.00, 100.00, 0.0) +_TUNNEL_B = (100.00, 97.79, -0.5) +_TUNNEL_C = (122.15, 97.79, -0.5) +_TUNNEL_D = (144.30, 97.79, -0.5) +_TUNNEL_E = (144.30, 100.00, 0.0) +_TUNNEL_HEADS = { + 1: (108.00, 100.00, 0.0), + 2: (126.00, 100.00, 0.0), + 3: (142.00, 100.00, 0.0), +} +_TUNNEL_DEPTH = 20.0 +# The trench cable is wound helically, so a meter of fiber covers cos(phi) +# of a meter of tunnel. +_TUNNEL_WIND = 0.886 +_TUNNEL_TRENCH = ( + "drop into the trench", + "trench B to the coil", + "trench from the coil to D", + "rise out of the trench", +) +_TUNNEL_REPAIRED_TRENCH = ( + "drop into the trench", + "trench B to the break", + "trench from the break to the coil", + "trench from the coil to D", + "rise out of the trench", +) + + +def _tunnel_spans(components_csv): + """Map each component's name to the optical interval it covers.""" + frame = pd.read_csv(io.StringIO(components_csv)) + end = frame["optical_length"].cumsum() + return dict(zip(frame["name"], zip(end - frame["optical_length"], end))) + + +def _tunnel_bottom(number): + """The bottom of a borehole is its head, straight down.""" + x, y, _ = _TUNNEL_HEADS[number] + return (x, y, -_TUNNEL_DEPTH) + + +def _tunnel_runs(repaired=False): + """Which component runs between which two surveyed waypoints.""" + if repaired: + # The patch cord is coiled in a splice box, so the trench is + # surveyed up to the break and again from it, and the two meters + # between get no position at all. + brk = (100.00 + 15.0 * _TUNNEL_WIND, 97.79, -0.5) + runs = [ + ("drop into the trench", _TUNNEL_A, _TUNNEL_B), + ("trench B to the break", _TUNNEL_B, brk), + ("trench from the break to the coil", brk, _TUNNEL_C), + ("trench from the coil to D", _TUNNEL_C, _TUNNEL_D), + ("rise out of the trench", _TUNNEL_D, _TUNNEL_E), + ] + else: + runs = [ + ("drop into the trench", _TUNNEL_A, _TUNNEL_B), + ("trench B to the coil", _TUNNEL_B, _TUNNEL_C), + ("trench from the coil to D", _TUNNEL_C, _TUNNEL_D), + ("rise out of the trench", _TUNNEL_D, _TUNNEL_E), + ] + for number in (3, 2, 1): + runs.append( + (f"borehole {number} down", _TUNNEL_HEADS[number], _tunnel_bottom(number)) + ) + runs.append( + (f"borehole {number} up", _tunnel_bottom(number), _TUNNEL_HEADS[number]) + ) + return runs + + +def _tunnel_geometry(at, runs): + """Turn each straight run into the two control points which place it.""" + rows = [] + for name, start, end in runs: + first, last = at[name] + rows.append((name, first, *start)) + rows.append((name, last, *end)) + frame = pd.DataFrame(rows, columns=["segment", "distance", "x", "y", "z"]) + return frame.to_csv(index=False) + + +def _tunnel_coupling(at, trench_parts): + """Buried in the trench, coiled at C, cemented in the boreholes.""" + rows: list[tuple] = [ + (*at[name], "trench", "soil", "direct_burial", 0.5) for name in trench_parts + ] + rows.append((*at["cable coil at C"], "coiled", "soil", "", 0.5)) + rows.extend( + ( + at[f"borehole {number} down"][0], + at[f"borehole {number} up"][1], + "outside_borehole_casing", + "rock", + "cemented", + "", + ) + for number in (3, 2, 1) + ) + frame = pd.DataFrame( + rows, + columns=[ + "start_distance", + "end_distance", + "coupling_type", + "medium", + "attachment", + "depth", + ], + ) + return frame.to_csv(index=False) + + +def _tunnel_labels(at, trench_parts): + """Which section a channel is in, and which borehole if it is in one.""" + rows: list[tuple] = [(*at[name], "section", "trench") for name in trench_parts] + rows.append((*at["cable coil at C"], "section", "coil")) + for number in (3, 2, 1): + span = (at[f"borehole {number} down"][0], at[f"borehole {number} up"][1]) + rows.append((*span, "section", "borehole")) + rows.append((*span, "borehole", number)) + frame = pd.DataFrame( + rows, columns=["start_distance", "end_distance", "group", "value"] + ) + return frame.to_csv(index=False) + + +def _tunnel_repaired_components(): + """One row becomes five where the contractor cut the trench cable.""" + rows = pd.read_csv(io.StringIO(_TUNNEL_COMPONENTS)).to_dict("records") + index = next( + i for i, row in enumerate(rows) if row["name"] == "trench B to the coil" + ) + rows[index : index + 1] = [ + dict( + object_type="FiberSegment", + optical_length=15.0, + name="trench B to the break", + container="trench-cable", + ), + dict( + object_type="Splice", + optical_length=0.0, + name="repair splice near side", + container="repair-box", + ), + dict( + object_type="FiberSegment", + optical_length=2.0, + name="repair patch cord", + container="repair-cord", + ), + dict( + object_type="Splice", + optical_length=0.0, + name="repair splice far side", + container="repair-box", + ), + dict( + object_type="FiberSegment", + optical_length=10.0, + name="trench from the break to the coil", + container="trench-cable", + ), + ] + frame = pd.DataFrame(rows) + frame["sequence"] = range(1, len(frame) + 1) + return frame.to_csv(index=False) + + +def tunnel_inventory_files(repaired: bool = True) -> dict[str, str]: + """ + Return the tunnel inventory as a mapping of file name to file text. + + This is the authoring directory the tunnel recipe writes, as data. It + is exposed so the recipe can display the same files the example + loads, rather than composing a second copy which could drift. + + Parameters + ---------- + repaired + Whether to include the epoch added when the trench cable was + repaired. False is the deployment as first installed, which is + what the recipe shows before it gets to the repair. + """ + array = "fiber_arrays/XT.TUN1" + path, epoch = f"{array}/path.00", f"{array}/path.00@2024-09-01" + at = _tunnel_spans(_TUNNEL_COMPONENTS) + repaired_csv = _tunnel_repaired_components() + repaired_at = _tunnel_spans(repaired_csv) + files = { + "inventory.yaml": ( + "object_type: Inventory\n" + "coordinate_reference_system:\n" + " authority: local\n" + " code: tunnel\n" + " name: tunnel engineering grid\n" + " coordinate_labels: [x, y, z]\n" + " units: [meter, meter, meter]\n" + ), + f"{array}/attrs.yaml": ("object_type: FiberArray\nname: tunnel fiber array\n"), + "acquisitions/XT.TUN1.00.DAS.yaml": ( + "object_type: Acquisition\n" + "data_category: DAS\n" + "data_type: strain_rate\n" + "data_units: 1/s\n" + "interrogator: das-interrogator\n" + "gauge_length: 10.0\n" + "spatial_interval: 1.0\n" + "sample_rate: 250.0\n" + "distance_map:\n" + " instrument_distance: [0.0, 2000.0]\n" + " distance: [0.0, 2000.0]\n" + ), + f"{path}/attrs.yaml": "object_type: OpticalPath\n", + f"{path}/optical_components.csv": _TUNNEL_COMPONENTS, + f"{path}/geometry.csv": _tunnel_geometry(at, _tunnel_runs()), + f"{path}/coupling.csv": _tunnel_coupling(at, _TUNNEL_TRENCH), + f"{path}/labels.csv": _tunnel_labels(at, _TUNNEL_TRENCH), + f"{epoch}/attrs.yaml": "object_type: OpticalPath\n", + f"{epoch}/optical_components.csv": repaired_csv, + f"{epoch}/geometry.csv": _tunnel_geometry( + repaired_at, _tunnel_runs(repaired=True) + ), + f"{epoch}/coupling.csv": _tunnel_coupling(repaired_at, _TUNNEL_REPAIRED_TRENCH), + f"{epoch}/labels.csv": _tunnel_labels(repaired_at, _TUNNEL_REPAIRED_TRENCH), + } + for name, text in _TUNNEL_RESOURCES.items(): + files[f"resources/{name}.yaml"] = text + if not repaired: + # The repair is later hardware, so before it happens neither its + # epoch nor the resources it introduced exist yet. + files = { + name: text + for name, text in files.items() + if epoch not in name and "repair-" not in name + } + return files + + +def write_tunnel_inventory(path, repaired: bool = True) -> Path: + """Write the tunnel inventory's authoring directory and return it.""" + path = Path(path) + for name, text in tunnel_inventory_files(repaired=repaired).items(): + file_path = path / name + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.write_text(text) + return path + + +@register_func(EXAMPLE_INVENTORIES, key="tunnel") +def tunnel_inventory() -> Inventory: + """The tunnel deployment the tunnel recipe builds, read from its files.""" + directory = Path(tempfile.mkdtemp()) / "tunnel_inventory" + return dc.inventory(write_tunnel_inventory(directory)) diff --git a/docs/contributing/adding_test_data.qmd b/docs/contributing/adding_test_data.qmd index 67b823a5e..c47c82ee4 100644 --- a/docs/contributing/adding_test_data.qmd +++ b/docs/contributing/adding_test_data.qmd @@ -6,7 +6,7 @@ There are a few different way to add test data to dascore. The key, however, is # Adding functions which create example data -The [examples module](`dascore.examples`) contains several functions for creating example `Patch` and `Spool` instances. You can add a new function in that module which creates a new patch or spool, then just register the function so it can be called from `dc.get_example_patch` or `dc.get_example_spool`. These should be simple objects which can be generated within python. If you need to download a file see +The [examples module](`dascore.examples`) contains several functions for creating example `Patch`, `Spool`, and `Inventory` instances. You can add a new function in that module which creates one, then just register the function so it can be called from `dc.get_example_patch`, `dc.get_example_spool`, or `dc.get_example_inventory`. These should be simple objects which can be generated within python. If you need to download a file see [adding a data file](#adding_a_data_file). :::{.callout-note} @@ -26,6 +26,11 @@ def create_example_patch(argument_1='default_value'): @register_func(EXAMPLE_SPOOLS, key="new_das_spool") def create_example_spool(another_value=None): ... + +# Register an example inventory function +@register_func(EXAMPLE_INVENTORIES, key="new_das_inventory") +def create_example_inventory(): + ... ``` The new example patches/spools can then be created via @@ -37,9 +42,11 @@ import dascore as dc patch_example = dc.get_example_patch("new_das_patch", argument_1="bob") spool_example = dc.get_example_spool("new_das_spool") + +inventory_example = dc.get_example_inventory("new_das_inventory") ``` -If, in the test code, the example patch or spool is used only once, just call the get_example function in the test. If it is needed multiple times, consider putting it in a fixture. See [testing](./testing.qmd) for more on fixtures. +If, in the test code, the example object is used only once, just call the get_example function in the test. If it is needed multiple times, consider putting it in a fixture. See [testing](./testing.qmd) for more on fixtures. # Adding a data file diff --git a/docs/recipes/tunnel_inventory.qmd b/docs/recipes/tunnel_inventory.qmd index c81e3a6a3..06d9a3efc 100644 --- a/docs/recipes/tunnel_inventory.qmd +++ b/docs/recipes/tunnel_inventory.qmd @@ -16,22 +16,33 @@ The inventory is written the way a field crew would keep it: a directory of smal import io import tempfile from pathlib import Path -from textwrap import dedent import numpy as np import pandas as pd import dascore as dc +from dascore.examples import tunnel_inventory_files, write_tunnel_inventory +# The files below are the deployment, held as data in dascore.examples so +# that this page and `dc.get_example_inventory("tunnel")` cannot drift +# apart. Every one of them is shown here as the example holds it. +# The deployment as first installed; the repair is added further down. +files = tunnel_inventory_files(repaired=False) root = Path(tempfile.mkdtemp()) / "tunnel_inventory" +write_tunnel_inventory(root, repaired=False) -def write(name, text): - """Write one file into the inventory directory.""" - path = root / name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(text) - return path +def show(name): + """Print one of the inventory's files.""" + print(files[name]) + + +def table(name): + """Read one of the inventory's CSV files back as a frame.""" + return pd.read_csv(io.StringIO(files[name])) + + +PATH = "fiber_arrays/XT.TUN1/path.00" ``` # What the drawing says @@ -72,62 +83,9 @@ Cables, enclosures, and interrogators are named once and referred to from everyw ```{python} #| code-fold: true #| code-summary: "resources/*.yaml — the hardware, from the purchase orders" -resources = { - "telemetry-cable": ( - "object_type: Cable\n" - "name: tunnel telemetry cable\n" - "manufacturer: Corning\n" - "model: MIC tight-buffered 4F OS2\n" - "fiber_count: 4\n" - "description: The run in from the instrument room.\n" - ), - "connecting-cable": ( - "object_type: Cable\n" - "name: tunnel connecting cable\n" - "manufacturer: Corning\n" - "model: MIC tight-buffered 4F OS2\n" - "fiber_count: 4\n" - "description: The links between boxes, couplers, and borehole heads.\n" - ), - "borehole-cable": ( - "object_type: Cable\n" - "name: borehole sensing cable\n" - "manufacturer: Nerve Sensors\n" - "model: Epsilon\n" - "fiber_count: 4\n" - "description: Rock-coupled downhole cable with an armored pigtail.\n" - ), - "trench-cable": ( - "object_type: Cable\n" - "name: helically wound trench cable\n" - "manufacturer: Silixa\n" - "model: HWC\n" - "fiber_count: 1\n" - ), - "das-interrogator": ( - "object_type: Interrogator\n" - "name: tunnel DAS interrogator\n" - "manufacturer: Sintela\n" - "model: Onyxia\n" - "instrument_type: DAS interrogator\n" - ), -} -# One enclosure per housing, because a resource_id names an asset rather -# than a kind: box A and box E are two boxes, and each borehole has its own -# turnaround down the hole. -for label, name in [ - ("splice-box-a", "splice box at A"), - ("splice-box-e", "splice box at E"), - ("turnaround-1", "borehole 1 turnaround housing"), - ("turnaround-2", "borehole 2 turnaround housing"), - ("turnaround-3", "borehole 3 turnaround housing"), -]: - kind = "box" if "splice" in label else "housing" - resources[label] = ( - f"object_type: Enclosure\nname: tunnel {name}\nenclosure_type: {kind}\n" - ) -for name, text in resources.items(): - write(f"resources/{name}.yaml", text) +for name in sorted(x for x in files if x.startswith("resources/")): + print(f"# {name}") + print(files[name]) ``` These stay objects rather than rows because they have nothing in common: a cable has a fiber count, an enclosure has an inner diameter, an interrogator has a serial number. A table of one row with twelve mostly-empty columns would be worse than the file. @@ -135,21 +93,8 @@ These stay objects rather than rows because they have nothing in common: a cable The envelope names the document and states the coordinate reference system everything is expressed in. This tunnel has its own survey grid rather than a global reference, so the axes are `x`, `y`, and `z` in meters, with `z` positive up. ```{python} -write( - "inventory.yaml", - "object_type: Inventory\n" - "coordinate_reference_system:\n" - " authority: local\n" - " code: tunnel\n" - " name: tunnel engineering grid\n" - " coordinate_labels: [x, y, z]\n" - " units: [meter, meter, meter]\n", -) - -write( - "fiber_arrays/XT.TUN1/attrs.yaml", - "object_type: FiberArray\nname: tunnel fiber array\n", -) +show("inventory.yaml") +show("fiber_arrays/XT.TUN1/attrs.yaml") ``` # The path, as a table @@ -161,41 +106,9 @@ This is @tbl-path with the parts named, and it goes in `path.00` — the directo ```{python} #| code-fold: true #| code-summary: "path.00/optical_components.csv" -components_csv = dedent("""\ -sequence,object_type,optical_length,name,container -1,FiberSegment,1500.0,telemetry lead-in,telemetry-cable -2,Splice,0.0,splice at box A,splice-box-a -3,FiberSegment,2.5,drop into the trench,trench-cable -4,FiberSegment,25.0,trench B to the coil,trench-cable -5,FiberSegment,10.0,cable coil at C,trench-cable -6,FiberSegment,25.0,trench from the coil to D,trench-cable -7,FiberSegment,2.5,rise out of the trench,trench-cable -8,Splice,0.0,splice at box E,splice-box-e -9,FiberSegment,15.0,link E to borehole 3,connecting-cable -10,FiberSegment,20.0,borehole 3 down,borehole-cable -11,Splice,0.0,borehole 3 turnaround,turnaround-3 -12,FiberSegment,20.0,borehole 3 up,borehole-cable -13,FiberSegment,15.0,link borehole 3 to coupler G,connecting-cable -14,Connector,0.0,coupler G, -15,FiberSegment,15.0,link coupler G to borehole 2,connecting-cable -16,FiberSegment,20.0,borehole 2 down,borehole-cable -17,Splice,0.0,borehole 2 turnaround,turnaround-2 -18,FiberSegment,20.0,borehole 2 up,borehole-cable -19,FiberSegment,15.0,link borehole 2 to coupler H,connecting-cable -20,Connector,0.0,coupler H, -21,FiberSegment,15.0,link coupler H to borehole 1,connecting-cable -22,FiberSegment,20.0,borehole 1 down,borehole-cable -23,Splice,0.0,borehole 1 turnaround,turnaround-1 -24,FiberSegment,20.0,borehole 1 up,borehole-cable -25,FiberSegment,15.0,link borehole 1 back to box A,connecting-cable -26,Terminator,0.0,path end, -""") - -PATH = "fiber_arrays/XT.TUN1/path.00" -write(f"{PATH}/attrs.yaml", "object_type: OpticalPath\n") -write(f"{PATH}/optical_components.csv", components_csv) +components_csv = files[f"{PATH}/optical_components.csv"] -pd.read_csv(root / PATH / "optical_components.csv").head(9) +table(f"{PATH}/optical_components.csv").head(9) ``` Every column but `sequence` is a field of the class the `object_type` column names, so the table is barely a format of its own — it is the objects, written the way rows of the same shape are worth writing. `sequence` belongs to the table, and is dropped once it has put the rows in order. The blank `container` on the terminator is a field left unset; the ones which are filled are `resource_id`s pointing back at the files above. @@ -224,47 +137,7 @@ The survey points are the ones lettered in @fig-tunnel. Each straight run of fib ```{python} #| code-fold: true #| code-summary: "path.00/geometry.csv — the survey points, paired with the fiber between them" -A = (100.00, 100.00, 0.0) -B = (100.00, 97.79, -0.5) -C = (122.15, 97.79, -0.5) -D = (144.30, 97.79, -0.5) -E = (144.30, 100.00, 0.0) -HEADS = {1: (108.00, 100.00, 0.0), 2: (126.00, 100.00, 0.0), 3: (142.00, 100.00, 0.0)} -DEPTH = 20.0 - - -def bottom(number): - """The bottom of a borehole is its head, straight down.""" - x, y, _ = HEADS[number] - return (x, y, -DEPTH) - - -def surveyed_runs(at): - """Which component runs between which two survey points.""" - runs = [ - ("drop into the trench", A, B), - ("trench B to the coil", B, C), - ("trench from the coil to D", C, D), - ("rise out of the trench", D, E), - ] - for number in (3, 2, 1): - runs.append((f"borehole {number} down", HEADS[number], bottom(number))) - runs.append((f"borehole {number} up", bottom(number), HEADS[number])) - return runs - - -def geometry_table(at, runs): - """Turn each straight run into its two control points.""" - rows = [] - for name, start, end in runs: - first, last = at[name] - rows.append((name, first, *start)) - rows.append((name, last, *end)) - return pd.DataFrame(rows, columns=["segment", "distance", "x", "y", "z"]) - - -geometry = geometry_table(at, surveyed_runs(at)) -geometry.to_csv(root / PATH / "geometry.csv", index=False) +geometry = table(f"{PATH}/geometry.csv") geometry.head(8) ``` @@ -280,45 +153,7 @@ Coupling is what decides whether a wiggle means anything, and it is an interval ```{python} #| code-fold: true #| code-summary: "path.00/coupling.csv" -def coupling_table(at, trench_parts): - """Buried in the trench, coiled at C, cemented in the boreholes.""" - rows = [ - (*at[name], "trench", "soil", "direct_burial", 0.5) for name in trench_parts - ] - rows.append((*at["cable coil at C"], "coiled", "soil", "", 0.5)) - rows.extend( - ( - at[f"borehole {number} down"][0], - at[f"borehole {number} up"][1], - "outside_borehole_casing", - "rock", - "cemented", - "", - ) - for number in (3, 2, 1) - ) - return pd.DataFrame( - rows, - columns=[ - "start_distance", - "end_distance", - "coupling_type", - "medium", - "attachment", - "depth", - ], - ) - - -TRENCH_PARTS = ( - "drop into the trench", - "trench B to the coil", - "trench from the coil to D", - "rise out of the trench", -) - -coupling = coupling_table(at, TRENCH_PARTS) -coupling.to_csv(root / PATH / "coupling.csv", index=False) +coupling = table(f"{PATH}/coupling.csv") coupling ``` @@ -332,21 +167,7 @@ Labels are for what the model has no field for. Each group becomes a coordinate ```{python} #| code-fold: true #| code-summary: "path.00/labels.csv" -def label_table(at, trench_parts): - """Which section a channel is in, and which borehole if it is in one.""" - rows = [(*at[name], "section", "trench") for name in trench_parts] - rows.append((*at["cable coil at C"], "section", "coil")) - for number in (3, 2, 1): - span = (at[f"borehole {number} down"][0], at[f"borehole {number} up"][1]) - rows.append((*span, "section", "borehole")) - rows.append((*span, "borehole", number)) - return pd.DataFrame( - rows, columns=["start_distance", "end_distance", "group", "value"] - ) - - -labels = label_table(at, TRENCH_PARTS) -labels.to_csv(root / PATH / "labels.csv", index=False) +labels = table(f"{PATH}/labels.csv") labels ``` @@ -360,20 +181,7 @@ A path is the fiber; an acquisition is a configuration of an instrument recordin The `distance_map` is the join between instrument and fiber: it says where the interrogator's own distance axis lands on the path. This one is zeroed at itself, so the map is the identity — but it is stated rather than assumed, because a re-zeroed instrument is the usual cause of metadata that is quietly off by a lead-in. It is also stated past the end of the fiber, so that it keeps working when the fiber gets longer. ```{python} -write( - "acquisitions/XT.TUN1.00.DAS.yaml", - "object_type: Acquisition\n" - "data_category: DAS\n" - "data_type: strain_rate\n" - "data_units: 1/s\n" - "interrogator: das-interrogator\n" - "gauge_length: 10.0\n" - "spatial_interval: 1.0\n" - "sample_rate: 250.0\n" - "distance_map:\n" - " instrument_distance: [0.0, 2000.0]\n" - " distance: [0.0, 2000.0]\n", -) +show("acquisitions/XT.TUN1.00.DAS.yaml") ``` The telemetry cable holds four fibers and only one is in use here. A second interrogator on another of them would be a second optical path under this same fiber array, at its own location code, with its own acquisitions — the array is the installation, not the instrument. @@ -450,36 +258,18 @@ This is what epochs are for. The old description is not edited. A second directo The components table is where the repair happens: one row becomes five, and the table is renumbered. ```{python} -# The repair is new hardware, so it is new resources: a patch cord, and the -# box holding the slack. -write( - "resources/repair-cord.yaml", - "object_type: Cable\nname: trench repair patch cord\nfiber_count: 1\n", -) -write( - "resources/repair-box.yaml", - "object_type: Enclosure\nname: trench repair box\nenclosure_type: box\n", -) +EPOCH = "fiber_arrays/XT.TUN1/path.00@2024-09-01" + +# The repair is new hardware, so it is new resources: a patch cord, and +# the box holding the slack. Both are already among the resources above. +# The repair is new hardware and a new epoch, so the file set grows. +files = tunnel_inventory_files(repaired=True) +write_tunnel_inventory(root, repaired=True) -rows = pd.read_csv(io.StringIO(components_csv)).to_dict("records") -index = next(i for i, row in enumerate(rows) if row["name"] == "trench B to the coil") -rows[index : index + 1] = [ - dict(object_type="FiberSegment", optical_length=15.0, - name="trench B to the break", container="trench-cable"), - dict(object_type="Splice", optical_length=0.0, - name="repair splice near side", container="repair-box"), - dict(object_type="FiberSegment", optical_length=2.0, - name="repair patch cord", container="repair-cord"), - dict(object_type="Splice", optical_length=0.0, - name="repair splice far side", container="repair-box"), - dict(object_type="FiberSegment", optical_length=10.0, - name="trench from the break to the coil", container="trench-cable"), -] - -repaired_components = pd.DataFrame(rows) -repaired_components["sequence"] = range(1, len(repaired_components) + 1) -repaired_csv = repaired_components.to_csv(index=False) +repaired_components = table(f"{EPOCH}/optical_components.csv") +repaired_csv = files[f"{EPOCH}/optical_components.csv"] +index = int(repaired_components.index[repaired_components["name"] == "trench B to the break"][0]) repaired_components.iloc[index - 1 : index + 5] ``` @@ -488,40 +278,17 @@ Everything else follows from that table, which is the argument for having writte ```{python} #| code-fold: true #| code-summary: "path.00@2024-09-01 — the same tracks, against the new distances" -EPOCH = "fiber_arrays/XT.TUN1/path.00@2024-09-01" -BREAK = (100.00 + 15.0 * 0.886, 97.79, -0.5) - repaired_at = spans(repaired_csv) -repaired_trench = ( - "drop into the trench", - "trench B to the break", - "trench from the break to the coil", - "trench from the coil to D", - "rise out of the trench", -) -# The patch cord is coiled in a splice box, so the trench is surveyed up to -# the break and again from it, and the two meters between get no position. -repaired_runs = [ - ("trench B to the break", B, BREAK), - ("trench from the break to the coil", BREAK, C), -] + [x for x in surveyed_runs(repaired_at) if x[0] != "trench B to the coil"] - -write(f"{EPOCH}/attrs.yaml", "object_type: OpticalPath\n") -write(f"{EPOCH}/optical_components.csv", repaired_csv) -geometry_table(repaired_at, repaired_runs).to_csv( - root / EPOCH / "geometry.csv", index=False -) -coupling_table(repaired_at, repaired_trench).to_csv( - root / EPOCH / "coupling.csv", index=False -) -label_table(repaired_at, repaired_trench).to_csv( - root / EPOCH / "labels.csv", index=False -) + +# The three other tracks are regenerated from the new running totals +# rather than edited row by row. +repaired_geometry = table(f"{EPOCH}/geometry.csv") +print(repaired_geometry.head(6).to_string(index=False)) repaired = dc.inventory(root) for epoch in repaired.networks[0].fiber_arrays[0].optical_paths: - start = "the beginning" if np.isnat(epoch.start_time) else str(epoch.start_time)[:10] - print(f"from {start}: {epoch.optical_length:.1f} m") + began = "the beginning" if np.isnat(epoch.start_time) else str(epoch.start_time)[:10] + print(f"from {began}: {epoch.optical_length:.1f} m") ``` A patch recorded across midnight on the first of September was recorded through both. [`Spool.conform_to_inventory`](`dascore.core.spool.Spool.conform_to_inventory`) is the step which insists every patch be describable by exactly one entry, and it subdivides that patch rather than choosing for you: diff --git a/docs/tutorial/inventory.qmd b/docs/tutorial/inventory.qmd index e0ecef734..2e985f148 100644 --- a/docs/tutorial/inventory.qmd +++ b/docs/tutorial/inventory.qmd @@ -161,6 +161,8 @@ from dascore.examples import inventory_patch_pair patch, example_inventory = inventory_patch_pair() ``` +Where a patch is not needed, [`dc.get_example_inventory`](`dascore.examples.get_example_inventory`) returns a registered inventory on its own, the same way `dc.get_example_patch` returns a patch. `dc.get_example_inventory("tunnel")` is the whole deployment the [tunnel recipe](../recipes/tunnel_inventory.qmd) builds, read from the same files that page shows. + # What an inventory can contribute [`Inventory.get_names`](`dascore.core.inventory.Inventory.get_names`) lists the names an inventory could put on a patch, split by where each one lands. `attrs` are the observing-system facts, which are one value per patch; `coords` take a value per channel, because they describe somewhere along the fiber. diff --git a/pyproject.toml b/pyproject.toml index 04461b8be..2225f6052 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,8 @@ dependencies = [ "pandas>=2.0", "pooch>=1.3", # retry_if_failed was added in 1.3 "pydantic>2.1", + # An inventory is authored as YAML, so reading one is not optional. + "pyyaml", "rich", "typing_extensions>=4.12", "universal-pathlib", @@ -91,8 +93,6 @@ docs = [ "jupyter-client", "nbclient", "nbformat", - # quarto's own notebook driver does `from yaml import safe_load`. - "pyyaml", # The JupyterLite site the tutorial pages link to. jupyter-server is not # used at runtime, but jupyterlite-core requires it to add custom content. "jupyterlite-core", diff --git a/tests/test_core/test_annotation_loader.py b/tests/test_core/test_annotation_loader.py index f8fea8bf4..238bb13d3 100644 --- a/tests/test_core/test_annotation_loader.py +++ b/tests/test_core/test_annotation_loader.py @@ -9,11 +9,7 @@ import numpy as np import pandas as pd import pytest - -try: - import yaml -except ImportError: - yaml = None +import yaml try: import pyarrow @@ -326,7 +322,6 @@ def test_a_stale_vertices_table_is_cleared(self, with_vertices, regions, tmp_pat assert not (directory / "vertices.csv").exists() assert dc.annotations(directory) == regions - @pytest.mark.skipif(yaml is None, reason="pyyaml is not installed") def test_a_hand_authored_yaml_is_superseded(self, tmp_path): """Saving a set read from YAML does not leave two attrs files.""" directory = tmp_path / "picks" @@ -494,7 +489,6 @@ def test_a_directory_which_states_none_takes_them(self, regions, tmp_path): class TestTheAttrsFile: """What a set directory says about itself.""" - @pytest.mark.skipif(yaml is None, reason="pyyaml is not installed") def test_yaml_spelling(self, regions, tmp_path): """One data model stands behind both spellings; a set may be authored in the more readable one. @@ -528,7 +522,6 @@ def test_which_is_not_a_mapping(self, regions, tmp_path): with pytest.raises(InvalidAnnotationError, match="no mapping"): dc.annotations(directory) - @pytest.mark.skipif(yaml is None, reason="pyyaml is not installed") def test_which_does_not_parse(self, regions, tmp_path): """Unparseable YAML names the file rather than the parser.""" directory = regions.io.save(tmp_path / "picks") diff --git a/tests/test_core/test_inventory.py b/tests/test_core/test_inventory.py index 2712784ad..3b6ccc52e 100644 --- a/tests/test_core/test_inventory.py +++ b/tests/test_core/test_inventory.py @@ -9,6 +9,7 @@ import numpy as np import pytest +import yaml from pydantic import ValidationError import dascore as dc @@ -1045,7 +1046,6 @@ def test_duplicate_network_codes_raise(self): def test_yaml_roundtrip(self, tmp_path): """Yaml roundtrip.""" - pytest.importorskip("yaml") inventory = build_inventory() path = tmp_path / "inventory.yaml" inventory.io.to_yaml(path) @@ -1077,7 +1077,6 @@ def test_replace_missing_raises(self): def test_dc_namespace(self, tmp_path): """Dc namespace.""" - pytest.importorskip("yaml") assert isinstance(dc.inventory(), inv.Inventory) path = tmp_path / "inv.yaml" build_inventory().io.to_yaml(path) @@ -1284,7 +1283,6 @@ def test_resource_correction_must_keep_id(self): def test_yaml_roundtrip_stays_flat(self, tmp_path): """Serialized form holds ids, not inline copies, and round-trips.""" - pytest.importorskip("yaml") cable = inv.Cable(resource_id="cable-01", name="c") seg = inv.FiberSegment(optical_length=100.0, container=cable) inventory = self._inventory_with(seg) @@ -1369,7 +1367,6 @@ def test_replace_finds_channel(self): def test_keyless_dict_resource_adopts_key(self): """A dict resource without resource_id adopts its pool key.""" - pytest.importorskip("yaml") inventory = inv.Inventory( resources={"cab-1": {"object_type": "Cable", "name": "mycable"}} ) @@ -1599,7 +1596,6 @@ def test_replace_missing_resource_raises(self): def test_from_yaml_non_mapping_raises(self): """From yaml non mapping raises.""" - pytest.importorskip("yaml") with pytest.raises(InvalidInventoryError, match="mapping"): inv.Inventory.from_yaml("- 1\n- 2\n") @@ -1665,7 +1661,6 @@ def test_description_on_previous_gaps(self): def test_yaml_omits_empty_fields(self): """Empty strings, dicts, and tuples do not serialize.""" - pytest.importorskip("yaml") inventory = build_inventory() text = inventory.io.to_yaml() assert "description:" not in text @@ -1675,7 +1670,6 @@ def test_yaml_omits_empty_fields(self): def test_extra_fields_contents_survive(self): """User values inside extra_fields are kept verbatim, even empty.""" - pytest.importorskip("yaml") acq = inv.Acquisition(code="RAW", extra_fields={"vendor_flag": ""}) array = inv.FiberArray(code="L001", acquisitions=(acq,)) inventory = inv.Inventory( @@ -1768,7 +1762,6 @@ def test_hash_survives_a_pickle_round_trip(self): def test_hash_survives_a_yaml_round_trip(self): """Serializing and reloading does not move an inventory's hash.""" - pytest.importorskip("yaml") inventory = self._stocked_inventory() loaded = inv.Inventory.from_yaml(inventory.io.to_yaml()) assert loaded == inventory @@ -2461,13 +2454,11 @@ class TestSerializationIsLossless: @pytest.mark.parametrize("name", sorted(SAMPLE_INVENTORIES)) def test_round_trip_equals(self, name): """Whatever was written comes back, in text and through a file.""" - pytest.importorskip("yaml") inventory = SAMPLE_INVENTORIES[name] assert dc.inventory(inventory.io.to_yaml()) == inventory def test_a_label_value_of_one_survives(self): """`1 == True`, and the value's default is True, so it was dropped.""" - pytest.importorskip("yaml") path = inv.OpticalPath( optical_components=(inv.FiberSegment(optical_length=100.0),), labels=( @@ -2507,7 +2498,6 @@ def test_a_deliberately_excluded_value_stays_out(self): def test_a_flag_label_stays_terse(self): """A value which really is the default is still left out.""" - pytest.importorskip("yaml") path = inv.OpticalPath( optical_components=(inv.FiberSegment(optical_length=100.0),), labels=( @@ -2526,7 +2516,6 @@ def test_a_flag_label_stays_terse(self): def test_round_trip_through_file(self, tmp_path): """The writer taking a path writes what the text form holds.""" - pytest.importorskip("yaml") inventory = build_full_inventory() path = tmp_path / "inventory.yaml" inventory.io.to_yaml(path) @@ -2534,14 +2523,12 @@ def test_round_trip_through_file(self, tmp_path): def test_blank_crs_fields_survive(self): """A frame described by WKT alone must not reload as EPSG:4979.""" - pytest.importorskip("yaml") inventory = SAMPLE_INVENTORIES["blank_crs"] crs = dc.inventory(inventory.io.to_yaml()).coordinate_reference_system assert (crs.authority, crs.code, crs.name) == ("", "", "") def test_blank_instrument_type_survives(self): """A blanked field with a non-empty default is not a missing one.""" - pytest.importorskip("yaml") inventory = SAMPLE_INVENTORIES["blank_resources"] loaded = dc.inventory(inventory.io.to_yaml()) assert loaded.resources["int-1"].instrument_type == "" @@ -2552,7 +2539,6 @@ def test_every_blankable_field_survives(self): Sweeping the model tree, rather than listing the fields at risk, is what catches the next field added with a non-empty default. """ - pytest.importorskip("yaml") inventory = build_full_inventory() checked = set() for model in _walk_models(inventory): @@ -2577,7 +2563,6 @@ def test_every_blankable_field_survives(self): def test_defaulted_fields_are_dropped(self): """A field still holding its default is left out of the document.""" - yaml = pytest.importorskip("yaml") data = yaml.safe_load(build_inventory().io.to_yaml()) assert not _empty_keys(data) assert "description" not in yaml.safe_dump(data) @@ -2588,7 +2573,6 @@ def test_document_states_its_schema_version(self): Every other defaulted field is dropped, so this is what says which envelope a document was written against. """ - yaml = pytest.importorskip("yaml") default = inv.Inventory().schema_version data = yaml.safe_load(build_inventory().io.to_yaml()) assert data["schema_version"] == default @@ -2614,7 +2598,6 @@ class TestLoadingValidates: def test_invalid_document_raises_on_load(self): """A document violating a whole-tree rule fails at its source.""" - pytest.importorskip("yaml") station = inv.Station(code="VA01", coordinates=(1.0, 2.0)) inventory = inv.Inventory( networks=(inv.Network(code="XX", stations=(station,)),) @@ -2894,13 +2877,11 @@ class TestInventoryNamespaces: def test_io_namespace(self): """The io namespace DASCore registers is reachable.""" - pytest.importorskip("yaml") inventory = build_inventory() assert inventory.io.to_yaml() == inv.inventory_to_yaml(inventory) def test_copy_gets_its_own_binding(self): """A copied inventory hands out a namespace bound to the copy.""" - pytest.importorskip("yaml") inventory = build_inventory() inventory.io # a namespace kept on the host would ride along other = inventory.model_copy(update={"schema_version": 7}) @@ -2938,7 +2919,6 @@ def test_unknown_private_attr_raises(self): def test_cached_namespace_is_not_state(self): """An attached namespace changes neither equality nor a dump.""" - pytest.importorskip("yaml") inventory = build_inventory() other = inv.Inventory(**inventory.model_dump()) inventory.io.to_yaml() diff --git a/tests/test_core/test_inventory_loader.py b/tests/test_core/test_inventory_loader.py index 85863379b..f677acf36 100644 --- a/tests/test_core/test_inventory_loader.py +++ b/tests/test_core/test_inventory_loader.py @@ -15,16 +15,10 @@ import dascore as dc from dascore.core import inventory as inv from dascore.core import inventory_loader as loader -from dascore.exceptions import ( - InvalidInventoryError, - MissingOptionalDependencyError, -) +from dascore.exceptions import InvalidInventoryError from dascore.models import InventoryModel, TimeRangedModel from dascore.models.registry import TAG_FIELD -pytest.importorskip("yaml") - - # A minimal directory which loads: one acquisition names everything above it. PATH_DIRECTORY = { "acquisitions/DAS.L001..RAW.yaml": "object_type: Acquisition\ndata_category: DAS\n", @@ -598,18 +592,12 @@ def test_envelope_stating_only_its_type(self, tmp_path): ) assert not dc.inventory(root).networks - def test_json_inventory_without_pyyaml(self, tmp_path, monkeypatch): + def test_json_inventory_beside_unrelated_yaml(self, tmp_path): """A JSON inventory loads past whatever YAML lies beside it. - PyYAML is optional, and the tests which need it skip without it, so - this pins the JSON-only path here rather than leaving it to a - minimal install nothing in this file would exercise. + The stray file is read, since deciding it declares no object is + what reading it is for, and it is then stepped over. """ - - def no_yaml(name, **kwargs): - raise MissingOptionalDependencyError(f"no {name}") - - monkeypatch.setattr(loader, "optional_import", no_yaml) root = write_inventory( tmp_path / "json_only", { @@ -2296,19 +2284,18 @@ def test_the_name_is_hidden(self): class TestLoadSerializedFile: """A whole inventory read from one document, not a directory.""" - def test_json_needs_no_yaml(self, tmp_path, monkeypatch): + def test_json_is_not_yamls_to_read(self, tmp_path, monkeypatch): """The suffix picks the parser, so JSON is not YAML's to read.""" path = tmp_path / "whole.json" path.write_text('{"description": "a JSON inventory"}') - def _refuse(name, **kwargs): - raise MissingOptionalDependencyError(name) + def refuse(*args, **kwargs): + raise AssertionError("the JSON route parsed YAML") - # Both bindings, because JSON is legal YAML: a fallback to the - # YAML route would parse this file and pass a test which only - # watched the loader's own import. - monkeypatch.setattr(loader, "optional_import", _refuse) - monkeypatch.setattr(inv, "optional_import", _refuse) + # JSON is legal YAML, so a fallback to the YAML route would parse + # this file and pass a test which only checked the result. + monkeypatch.setattr(loader.yaml, "safe_load", refuse) + monkeypatch.setattr(inv.yaml, "safe_load", refuse) assert dc.inventory(path).description == "a JSON inventory" def test_a_document_which_does_not_parse(self, tmp_path): diff --git a/tests/test_examples.py b/tests/test_examples.py index 14fefb41f..5a3aff784 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -3,12 +3,14 @@ from __future__ import annotations import numpy as np +import pandas as pd import pytest import dascore as dc import dascore.examples as dc_examples -from dascore.examples import EXAMPLE_PATCHES +from dascore.examples import EXAMPLE_INVENTORIES, EXAMPLE_PATCHES from dascore.exceptions import UnknownExampleError +from dascore.utils.intervals import normalize_value, value_kind from dascore.utils.time import to_float @@ -101,6 +103,114 @@ def test_data_file_name(self): assert isinstance(spool, dc.BaseSpool) +class TestGetExampleInventory: + """Test suite for `get_example_inventory`.""" + + def test_default(self): + """Ensure calling get_example_inventory with no args returns one.""" + inventory = dc.get_example_inventory() + assert isinstance(inventory, dc.Inventory) + + def test_raises_on_bad_key(self): + """Ensure a bad key raises expected error.""" + with pytest.raises(UnknownExampleError, match="No example inventory"): + dc.get_example_inventory("NotAnExampleRight????") + + @pytest.mark.parametrize("name", EXAMPLE_INVENTORIES) + def test_load_example_inventory(self, name): + """Each registered inventory loads and passes its own checks.""" + inventory = dc.get_example_inventory(name) + assert isinstance(inventory, dc.Inventory) + # check returns self, so this both validates and pins that. + assert inventory.check() == inventory + + +class TestTunnelInventory: + """The tunnel example is the deployment the tunnel recipe builds.""" + + @pytest.fixture(scope="class") + def inventory(self): + """The tunnel example inventory.""" + return dc.get_example_inventory("tunnel") + + @pytest.fixture(scope="class") + def original(self, inventory): + """The optical path as it was before the repair.""" + return inventory.networks[0].fiber_arrays[0].optical_paths[0] + + def test_repair_is_two_epochs_of_one_location(self, inventory): + """One location code carries two paths which do not overlap.""" + paths = inventory.networks[0].fiber_arrays[0].optical_paths + assert len(paths) == 2 + assert len({x.location_code for x in paths}) == 1 + assert not paths[0].overlaps(paths[1]) + # The repair spliced two meters of patch cord in. + assert paths[1].optical_length == paths[0].optical_length + 2.0 + + def test_epochs_are_open_at_the_ends(self, inventory): + """The first path runs from the beginning, the second is ongoing.""" + first, second = inventory.networks[0].fiber_arrays[0].optical_paths + assert pd.isnull(first.start_time) + assert first.end_time == second.start_time + assert pd.isnull(second.end_time) + + def test_geometry_gap_is_a_real_gap(self, inventory, original): + """Fiber nobody surveyed gets no position rather than a guess.""" + crs = inventory.coordinate_reference_system + # 1000 m along is slack cable in a tray; 1590 m is down borehole 3. + coords = original.coordinates_at(np.array([1000.0, 1590.0]), crs) + assert np.isnan(coords[0]).all() + assert coords[1][2] == -10.0 + + def test_every_label_group_appears(self, original): + """A string group and a numeric one, which color differently.""" + kinds = {} + for label in original.labels: + kinds.setdefault(label.group, set()).add( + value_kind(normalize_value(label.value)) + ) + assert kinds == {"section": {"string"}, "borehole": {"numeric"}} + + def test_holds_point_markers(self, original): + """Splices and connectors have no length, so they are points.""" + intervals = original.component_intervals() + assert sum(1 for start, end in intervals if start == end) > 1 + + def test_coupling_covers_only_part_of_the_path(self, original): + """Partial coverage is legal and is what a coverage plot must show.""" + assert len({x.coupling_type for x in original.coupling}) == 3 + covered = sum(x.optical_length for x in original.coupling) + assert covered < original.optical_length + + def test_files_are_the_recipe_directory(self): + """The example is its authoring files, which the recipe displays.""" + original = dc_examples.tunnel_inventory_files(repaired=False) + repaired = dc_examples.tunnel_inventory_files(repaired=True) + assert set(original) < set(repaired) + # Before the repair neither its epoch nor its hardware exists. + assert not any("@" in x for x in original) + assert any("@2024-09-01" in x for x in repaired) + + def test_written_directory_reads_back(self, tmp_path, inventory): + """Writing the files and reading them describes the same system.""" + path = dc_examples.write_tunnel_inventory(tmp_path / "tunnel") + loaded = dc.inventory(path) + # Not == : an Inventory carries its own resource_id, which is a + # fresh uuid on each read, so two reads of one directory differ + # in their document identity and in nothing else. + assert loaded.networks == inventory.networks + assert loaded.resources == inventory.resources + assert ( + loaded.coordinate_reference_system == inventory.coordinate_reference_system + ) + + def test_unrepaired_directory_holds_one_epoch(self, tmp_path): + """The deployment as installed is one path, not two.""" + path = dc_examples.write_tunnel_inventory(tmp_path / "first", repaired=False) + loaded = dc.inventory(path) + assert len(loaded.networks[0].fiber_arrays[0].optical_paths) == 1 + + class TestRickerMoveout: """Tests for Ricker moveout patch."""