diff --git a/dascore/core/_spool_inventory.py b/dascore/core/_spool_inventory.py index 27be9dc65..bb8190467 100644 --- a/dascore/core/_spool_inventory.py +++ b/dascore/core/_spool_inventory.py @@ -33,8 +33,6 @@ VALID_COORDINATE_LABELS, Inventory, ResolvedContext, - _annotation_kind, - interval_masks, ) from dascore.core.inventory_loader import BLESSED_NAME, find_inventory from dascore.exceptions import ( @@ -47,6 +45,7 @@ UnresolvedPatchError, ) from dascore.units import get_quantity_str +from dascore.utils.intervals import interval_masks, value_kind from dascore.utils.misc import iterate # One vocabulary for both fiber verbs. What the quiet option leaves @@ -625,7 +624,7 @@ def _get_annotation_coord(path, group, distances): items = [x for x in path.annotations if x.group == group] if not items: return None - kind = _annotation_kind(items[0].value) + kind = value_kind(items[0].value) intervals = [x.interval for x in items] values = [x.value for x in items] return _fill_from_intervals(distances, intervals, values, kind) @@ -648,7 +647,7 @@ def _get_track_coord(path, track, field, distances): # inventory defines nothing here, and on_missing then rules -- # rather than handing back a coordinate that is blank throughout. return None - kinds = {_annotation_kind(x) for x in values if not is_unset(x)} + kinds = {value_kind(x) for x in values if not is_unset(x)} kind = kinds.pop() if len(kinds) == 1 else "string" filled = _fill_from_intervals(distances, intervals, values, kind) if units := _TRACK_FIELD_UNITS.get(f"{track}.{field}"): diff --git a/dascore/core/inventory.py b/dascore/core/inventory.py index ebc99797a..45f87269a 100644 --- a/dascore/core/inventory.py +++ b/dascore/core/inventory.py @@ -51,6 +51,13 @@ TimeRangedModel, UnitQuantity, ) +from dascore.utils.intervals import ( + clip_intervals, + interval_masks, + intervals_overlap, + normalize_value, + value_kind, +) from dascore.utils.mapping import FrozenDict from dascore.utils.misc import ( check_code, @@ -107,18 +114,8 @@ def _annotation_value(value): - """ - Normalize an annotation value so its Python type survives validation. - - Numpy scalars are unwrapped: pydantic's smart union resolves every numpy - scalar to float, which would turn a mask element into a numeric group. - """ - if isinstance(value, np.generic): - value = value.item() - if isinstance(value, float) and not np.isfinite(value): - msg = f"Annotation value must be finite; got {value}." - raise InvalidInventoryError(msg) - return value + """Normalize an annotation value so its Python type survives validation.""" + return normalize_value(value, error=InvalidInventoryError) # The value kind decides an annotation group's shape, so it must be exact. @@ -574,31 +571,6 @@ def interpolate(self, distances) -> np.ndarray: return out -def interval_masks(values, intervals) -> list[np.ndarray]: - """ - Return, per interval, the mask of values that interval covers. - - Coverage is half-open, ``[start, end)``, with one exception: the end of - a coverage run belongs to the interval ending there when no half-open - interval claims it, so the last point of a run is not left out. Point - markers (equal start and end) cover nothing. - """ - values = np.asarray(values, dtype=float) - spans = [(lo, hi) for lo, hi in intervals] - claimed = np.zeros(len(values), dtype=bool) - for lo, hi in spans: - if lo < hi: - claimed |= (values >= lo) & (values < hi) - out = [] - for lo, hi in spans: - if lo >= hi: # a point marker covers nothing - out.append(np.zeros(len(values), dtype=bool)) - continue - mask = (values >= lo) & (values < hi) - out.append(mask | ((values == hi) & ~claimed)) - return out - - class _IntervalModel(InventoryModel): """ Base for items covering the half-open interval [start, end) of optical @@ -1062,27 +1034,6 @@ def _times_equal(time1, time2) -> bool: return bool(time1 == time2) -def _annotation_kind(value) -> str: - """Return the value kind which decides an annotation group's shape.""" - if isinstance(value, bool): # bool before int; bool is an int subclass - return "boolean" - if isinstance(value, str): - return "string" - return "numeric" - - -def _intervals_overlap(intervals: list[tuple[float, float]]) -> tuple | None: - """Return the first overlapping pair of half-open intervals, or None. - - Empty (point) intervals cover nothing and cannot overlap. - """ - ordered = sorted(x for x in intervals if x[0] < x[1]) - for first, second in itertools.pairwise(ordered): - if second[0] < first[1]: - return first, second - return None - - class OpticalPath(TimeRangedModel): """ Continuous optical path described by independent tracks. @@ -1203,7 +1154,7 @@ def check(self, tolerance: float = 1e-9) -> Self: if len(dims) > 1: errors.append(_MIXED_DIMS_MSG.format(dims=sorted(dims))) for name, spans in (("geometry", geo_spans), ("coupling", coup_spans)): - overlap = _intervals_overlap(spans) + overlap = intervals_overlap(spans) if overlap is not None: errors.append( f"Overlapping {name} intervals {overlap[0]} and " @@ -1228,7 +1179,7 @@ def _check_annotation_groups(self) -> list[str]: "coordinate, a typed track, or a coordinate label." ) for group, items in groups.items(): - kinds = {_annotation_kind(x.value) for x in items} + kinds = {value_kind(x.value) for x in items} if len(kinds) > 1: errors.append( f"Annotation group {group!r} mixes {sorted(kinds)} values; " @@ -1237,7 +1188,7 @@ def _check_annotation_groups(self) -> list[str]: continue if kinds == {"boolean"}: # membership groups may overlap continue - overlap = _intervals_overlap([x.interval for x in items]) + overlap = intervals_overlap([x.interval for x in items]) if overlap is not None: errors.append( f"Overlapping intervals {overlap[0]} and {overlap[1]} in " @@ -1295,8 +1246,8 @@ def select(self, *, distance: tuple[float | None, float | None]) -> Self: ) ) outer = self.end_distance - coupling = _clip_intervals(self.coupling, lo, hi, outer) - annotations = _clip_intervals(self.annotations, lo, hi, outer) + coupling = clip_intervals(self.coupling, lo, hi, outer) + annotations = clip_intervals(self.annotations, lo, hi, outer) return self.model_copy( update={ "start_distance": lo, @@ -1422,29 +1373,6 @@ def shift_item(item): ) -def _clip_intervals(items, lo: float, hi: float, outer: float | None = None) -> list: - """ - Clip interval items to [lo, hi), dropping those left with no coverage. - - Point markers cover nothing but are not nothing: they survive when they - fall inside the clip, or on its outermost included endpoint. - """ - out = [] - for item in items: - start, end = item.interval - if start == end: - if lo <= start < hi or (outer is not None and start == hi == outer): - out.append(item) - continue - new_lo, new_hi = max(start, lo), min(end, hi) - if new_hi <= new_lo: - continue - out.append( - item.model_copy(update={"start_distance": new_lo, "end_distance": new_hi}) - ) - return out - - class Response(InventoryModel): """Station-specific response model associated with a channel.""" diff --git a/dascore/core/inventory_loader.py b/dascore/core/inventory_loader.py index 4c7a4e132..5377f2360 100644 --- a/dascore/core/inventory_loader.py +++ b/dascore/core/inventory_loader.py @@ -30,7 +30,6 @@ from __future__ import annotations -import csv import itertools import json import os @@ -61,10 +60,20 @@ from dascore.exceptions import ( InvalidInventoryError, MissingOptionalDependencyError, + ParameterError, ) from dascore.models import InventoryModel, TimeRangedModel from dascore.models.registry import TAG_FIELD from dascore.utils.misc import check_code, optional_import +from dascore.utils.paths import quote_path as _quote +from dascore.utils.tables import ( + ordered_rows, + parse_cell, + read_table, + require_columns, + require_stated, + row_cells, +) from dascore.utils.time import to_datetime64 # One data model stands behind all three spellings, so they are accepted @@ -168,16 +177,6 @@ def walk(model): return frozenset(walk(InventoryModel)) | {Inventory.__name__} -def _quote(path: Path) -> str: - """ - Name a file for an error message. - - Its container is included because a bare name is ambiguous across - containers and the full path is noise the reader already knows. - """ - return str(Path(path.parent.name) / path.name) - - def _object_suffix(path: Path) -> str | None: """ Return a path's object-file suffix, or None if it has none. @@ -497,154 +496,6 @@ class _Table(NamedTuple): _SEQUENCE = "sequence" -def _read_table(path: Path) -> pd.DataFrame: - """ - Read one track table. - - Every cell arrives as text and the models coerce it, so a column's - meaning is the field's rather than whatever pandas inferred from the - rows it happened to see. Only a truly empty cell is null: an empty - cell means unset, and a document which writes ``NA`` means the string. - """ - # The header is read first and by itself, for two reasons: pandas - # renames a repeated column rather than refusing it, so by the time a - # frame exists the second one is `coupling_type.1` and the clash cannot - # be seen; and it raises its own error for a file with no columns, - # which would arrive before this one could say what was expected. - try: - with path.open(newline="", encoding="utf-8-sig") as stream: - reader = csv.reader(stream) - header = next(reader, []) - if header: - # Streamed rather than listed: a track table is the part of - # this format meant to grow, and holding every cell as a - # python object beside the frame pandas builds would cost - # several times what the frame itself does. - _check_widths(reader, header, path) - except (OSError, UnicodeDecodeError) as error: - msg = f"Could not read {_quote(path)}: {error}." - raise InvalidInventoryError(msg) from error - if not header: - msg = f"{_quote(path)} has no columns, so it states no track." - raise InvalidInventoryError(msg) - repeated = sorted({x for x in header if header.count(x) > 1}) - if repeated: - msg = ( - f"{_quote(path)} names {', '.join(repeated)} more than once; one " - "column states one field." - ) - raise InvalidInventoryError(msg) - # index_col=False so that no column is ever read as an index; the row - # widths above already agree, and this keeps them agreeing. - return pd.read_csv( - path, - dtype=str, - keep_default_na=False, - na_values=[""], - index_col=False, - # Both readers decode alike, or the header checked above is not - # the header parsed here: a locale-encoded read disagrees with - # pandas' UTF-8, and a byte order mark reaches only one of them. - encoding="utf-8-sig", - ) - - -def _check_widths(reader, header: list[str], path: Path) -> None: - """ - Refuse a row which is not its header wide. - - Pandas refuses neither a wide row nor a narrow one: by default the - surplus cell pushes the first column into the index, so every value in - the row shifts one field left and lands in its neighbour's meaning. A - row states one cell per column or it is not a row. - """ - for number, row in enumerate(reader, start=2): - if row and len(row) != len(header): - msg = ( - f"{_quote(path)} row {number} states {len(row)} cells where " - f"its header names {len(header)} columns." - ) - raise InvalidInventoryError(msg) - - -def _cells(row) -> dict[str, str]: - """Return a row's stated cells, an empty one meaning unset.""" - return {str(k): v for k, v in row.items() if not pd.isnull(v)} - - -def _require_columns(frame: pd.DataFrame, needed, path: Path) -> None: - """Refuse a table which does not carry a column it is read by.""" - missing = [x for x in needed if x is not None and x not in frame.columns] - if missing: - msg = ( - f"{_quote(path)} states no {', '.join(missing)} column, which its " - "rows are read by." - ) - raise InvalidInventoryError(msg) - - -def _require_stated(frame: pd.DataFrame, needed, path: Path) -> None: - """ - Refuse a blank cell in a column the table is read by. - - A column which orders or groups the rows decides where each one goes, - so a row leaving it empty has no place. Left to pandas the row would - simply disappear -- a null sorts last, and a null grouping key drops - its row from every group. - """ - for column in needed: - if column is None: - continue - empty = [ - str(n) for n, ok in enumerate(frame[column].notna(), start=2) if not ok - ] - if empty: - msg = ( - f"{_quote(path)} leaves {column} empty at row(s) " - f"{', '.join(empty)}, so those rows state no place." - ) - raise InvalidInventoryError(msg) - - -def _ordered(frame: pd.DataFrame, column: str | None, path: Path) -> pd.DataFrame: - """ - Return the rows in the order the named column states, if any. - - A table which names one is read by it rather than by row position, so - re-sorting a spreadsheet cannot change what it means. A table which - names none keeps the order it was written in. - """ - if column is None: - return frame - try: - keys = pd.to_numeric(frame[column]) - except (TypeError, ValueError) as error: - msg = f"{_quote(path)} has a non-numeric {column}: {error}." - raise InvalidInventoryError(msg) from error - return frame.assign(**{column: keys}).sort_values(column, kind="stable") - - -def _parse_cell(text: str): - """ - Read a cell's value the way its own text states it. - - A CSV has no types, so an annotation's value -- which the model lets - be a string, a boolean or a number -- is decided by what was written. - A value which is genuinely a string but looks like one of the others - is the one thing this spelling cannot express; that group is authored - in YAML, where the types are explicit. - """ - if (folded := text.strip().casefold()) in ("true", "false"): - return folded == "true" - try: - number = float(text) - except ValueError: - return text - # int(number) rather than int(text): 1e3 is integral, and only the - # number knows that -- the text raises. - return int(number) if number.is_integer() and "." not in text else number - - def _check_places(keys: pd.Series, column: str, path: Path) -> None: """ Refuse an ordering which does not place every row. @@ -665,14 +516,14 @@ def _check_places(keys: pd.Series, column: str, path: Path) -> None: def _object_rows(frame: pd.DataFrame, table: _Table, path: Path) -> list[dict]: """Read a table whose every row is one object.""" - _require_columns(frame, [table.order], path) - _require_stated(frame, [table.order], path) - ordered = _ordered(frame, table.order, path) + require_columns(frame, [table.order], path) + require_stated(frame, [table.order], path) + ordered = ordered_rows(frame, table.order, path) if table.places and table.order is not None: _check_places(ordered[table.order], table.order, path) out = [] for _, row in ordered.iterrows(): - cells = _cells(row) + cells = row_cells(row) # The order column is the table's own scaffolding where the object # has no such field, so it is dropped -- but only where the table # says it has one. Dropped everywhere, a stray sequence column in @@ -693,11 +544,11 @@ def _point_rows(frame: pd.DataFrame, table: _Table, path: Path, axes) -> list[di is one. Coordinate columns are named by the CRS and are stored on the canonical axes, so the frame decides which column is which. """ - _require_columns(frame, [table.order, table.group], path) - _require_stated(frame, [table.order, table.group], path) - frame = _ordered(frame, table.order, path) + require_columns(frame, [table.order, table.group], path) + require_stated(frame, [table.order, table.group], path) + frame = ordered_rows(frame, table.order, path) # dropna=False: a blank grouping cell would otherwise take its row out - # of the table without a word. _require_stated has already refused one, + # of the table without a word. require_stated has already refused one, # and this keeps that the reason nothing is missing. groups = ( frame.groupby(table.group, sort=True, dropna=False) @@ -962,8 +813,22 @@ def _merge_tables(data: dict, entity: Path, model, crs, attrs: Path) -> None: def _load_table(path: Path, table: _Table, stem: str, crs): - """Read one track table into whatever its attribute holds.""" - frame = _read_table(path) + """ + Read one track table into whatever its attribute holds. + + The table utilities are format-neutral, so this is where their errors + become the inventory's: one boundary rather than an exception class + threaded through every call. + """ + try: + return _read_track_table(path, table, stem, crs) + except ParameterError as error: + raise InvalidInventoryError(str(error)) from error + + +def _read_track_table(path: Path, table: _Table, stem: str, crs): + """Read one track table, in the table utilities' own error vocabulary.""" + frame = read_table(path, what="no track") # Refused here rather than left to the model: a header with nothing # under it claims a track and states none, and for a single-object # table it would otherwise build one object out of no points. @@ -1013,7 +878,7 @@ def _parse_annotations(rows: list[dict], path: Path) -> None: for number, row in enumerate(rows, start=2): if (text := row.get("value")) is None: continue - row["value"] = value = _parse_cell(text) + row["value"] = value = parse_cell(text) # A boolean is asked about first because a bool IS an int, which # would otherwise let true and 1 share a group whose shape they do # not share. An int and a float, by contrast, are ONE kind: the diff --git a/dascore/utils/intervals.py b/dascore/utils/intervals.py new file mode 100644 index 000000000..010b36873 --- /dev/null +++ b/dascore/utils/intervals.py @@ -0,0 +1,188 @@ +""" +Utilities for half-open intervals and the values they carry. + +Intervals are ``[start, end)`` everywhere: an interval whose start equals +its end is a point marker which covers nothing. These helpers are shared +by anything which lays values along an axis -- the inventory's optical +distance tracks, and the annotation sets which describe patch data. +""" + +from __future__ import annotations + +import itertools + +import numpy as np + +from dascore.exceptions import ParameterError + + +def interval_masks(values, intervals) -> list[np.ndarray]: + """ + Return, per interval, the mask of values that interval covers. + + Coverage is half-open, ``[start, end)``, with one exception: the end of + a coverage run belongs to the interval ending there when no half-open + interval claims it, so the last point of a run is not left out. Point + markers (equal start and end) cover nothing. + + Parameters + ---------- + values + The values, along the interval axis, to test for coverage. + intervals + A sequence of (start, end) pairs. + + Examples + -------- + >>> from dascore.utils.intervals import interval_masks + >>> masks = interval_masks([0, 1, 2, 3], [(0, 2), (2, 3)]) + >>> masks[0] + array([ True, True, False, False]) + >>> masks[1] # the run ends at 3, so 3 is not left uncovered + array([False, False, True, True]) + """ + values = np.asarray(values, dtype=float) + spans = [(lo, hi) for lo, hi in intervals] + claimed = np.zeros(len(values), dtype=bool) + for lo, hi in spans: + if lo < hi: + claimed |= (values >= lo) & (values < hi) + out = [] + for lo, hi in spans: + if lo >= hi: # a point marker covers nothing + out.append(np.zeros(len(values), dtype=bool)) + continue + mask = (values >= lo) & (values < hi) + out.append(mask | ((values == hi) & ~claimed)) + return out + + +def intervals_overlap(intervals) -> tuple | None: + """ + Return the first overlapping pair of half-open intervals, or None. + + Empty (point) intervals cover nothing and cannot overlap. + + Parameters + ---------- + intervals + A sequence of (start, end) pairs. + + Examples + -------- + >>> from dascore.utils.intervals import intervals_overlap + >>> intervals_overlap([(0, 2), (2, 4)]) is None + True + >>> intervals_overlap([(0, 3), (2, 4)]) + ((0, 3), (2, 4)) + """ + ordered = sorted(x for x in intervals if x[0] < x[1]) + for first, second in itertools.pairwise(ordered): + if second[0] < first[1]: + return first, second + return None + + +def clip_intervals( + items, + lo: float, + hi: float, + outer: float | None = None, + start_field: str = "start_distance", + end_field: str = "end_distance", +) -> list: + """ + Clip interval items to [lo, hi), dropping those left with no coverage. + + Point markers cover nothing but are not nothing: they survive when they + fall inside the clip, or on its outermost included endpoint. + + Parameters + ---------- + items + Models whose start and end are held by ``start_field`` and + ``end_field``, and which support pydantic's ``model_copy``. + lo + The inclusive start of the clip. + hi + The exclusive end of the clip. + outer + The outermost value the clipped axis holds, if any. A point marker + sitting exactly there survives, since nothing beyond it can claim it. + start_field + Name of the field holding each item's interval start. + end_field + Name of the field holding each item's interval end. + + Examples + -------- + >>> from pydantic import BaseModel + >>> from dascore.utils.intervals import clip_intervals + >>> class Span(BaseModel): + ... start_distance: float + ... end_distance: float + >>> clipped = clip_intervals([Span(start_distance=0, end_distance=10)], 2, 6) + >>> clipped[0].start_distance, clipped[0].end_distance + (2, 6) + """ + out = [] + for item in items: + start, end = getattr(item, start_field), getattr(item, end_field) + if start == end: + if lo <= start < hi or (outer is not None and start == hi == outer): + out.append(item) + continue + new_lo, new_hi = max(start, lo), min(end, hi) + if new_hi <= new_lo: + continue + out.append(item.model_copy(update={start_field: new_lo, end_field: new_hi})) + return out + + +def value_kind(value) -> str: + """ + Return the value kind which decides an interval group's shape. + + Boolean groups state membership and may overlap; string and numeric + groups are single valued where they project onto a coordinate. + + Examples + -------- + >>> from dascore.utils.intervals import value_kind + >>> value_kind(True), value_kind("car"), value_kind(1) + ('boolean', 'string', 'numeric') + """ + if isinstance(value, bool): # bool before int; bool is an int subclass + return "boolean" + if isinstance(value, str): + return "string" + return "numeric" + + +def normalize_value(value, error: type[Exception] = ParameterError): + """ + Normalize an interval's value so its Python type survives validation. + + Numpy scalars are unwrapped: pydantic's smart union resolves every numpy + scalar to float, which would turn a mask element into a numeric group. + + Parameters + ---------- + value + The value to normalize. + error + The exception raised when the value is a non-finite number. + + Examples + -------- + >>> import numpy as np + >>> from dascore.utils.intervals import normalize_value + >>> normalize_value(np.bool_(True)) is True + True + """ + if isinstance(value, np.generic): + value = value.item() + if isinstance(value, float) and not np.isfinite(value): + msg = f"Annotation value must be finite; got {value}." + raise error(msg) + return value diff --git a/dascore/utils/paths.py b/dascore/utils/paths.py index 12897799c..401610c37 100644 --- a/dascore/utils/paths.py +++ b/dascore/utils/paths.py @@ -30,6 +30,23 @@ def is_pathlike(resource) -> TypeIs[str | Path | UPath]: return isinstance(resource, str | Path | UPath) +def quote_path(path: Path) -> str: + """ + Name a file for an error message. + + Its container is included because a bare name is ambiguous across + containers and the full path is noise the reader already knows. + + Examples + -------- + >>> from pathlib import Path + >>> from dascore.utils.paths import quote_path + >>> Path(quote_path(Path("inventory/path@2020-01-01/coupling.csv"))).parts + ('path@2020-01-01', 'coupling.csv') + """ + return str(Path(path.parent.name) / path.name) + + def is_memory_uri(path) -> bool: """ Return True if a path is a synthetic in-memory patch identity. diff --git a/dascore/utils/tables.py b/dascore/utils/tables.py new file mode 100644 index 000000000..6110cbf49 --- /dev/null +++ b/dascore/utils/tables.py @@ -0,0 +1,264 @@ +""" +Utilities for reading strict CSV tables. + +A table written by hand is read the way it was written: every cell arrives +as text, the reader refuses a row which is not its header wide, and the +value of a cell is decided by what the cell says rather than by whatever +pandas inferred from the rows it happened to see. + +These raise `ParameterError` and name the file with +[quote_path](`dascore.utils.paths.quote_path`); a format which wants its own +error type wraps them once, at whatever boundary reads its tables. +""" + +from __future__ import annotations + +import csv +from pathlib import Path + +import pandas as pd + +from dascore.exceptions import ParameterError +from dascore.utils.paths import quote_path + + +def read_table(path: Path, what: str = "nothing") -> pd.DataFrame: + r""" + Read one strict CSV table. + + Every cell arrives as text and the caller coerces it, so a column's + meaning is the field's rather than whatever pandas inferred from the + rows it happened to see. Only a truly empty cell is null: an empty + cell means unset, and a document which writes ``NA`` means the string. + + Parameters + ---------- + path + The CSV file to read. + what + What a table with no columns fails to state, for that error message. + + Examples + -------- + >>> import tempfile + >>> from pathlib import Path + >>> from dascore.utils.tables import read_table + >>> with tempfile.TemporaryDirectory() as folder: + ... path = Path(folder) / "coupling.csv" + ... _ = path.write_text("group,value\nrail,true\n") + ... frame = read_table(path) + >>> list(frame.columns), frame["value"][0] + (['group', 'value'], 'true') + """ + # The header is read first and by itself, for two reasons: pandas + # renames a repeated column rather than refusing it, so by the time a + # frame exists the second one is `coupling_type.1` and the clash cannot + # be seen; and it raises its own error for a file with no columns, + # which would arrive before this one could say what was expected. + try: + with path.open(newline="", encoding="utf-8-sig") as stream: + reader = csv.reader(stream) + header = next(reader, []) + if header: + # Streamed rather than listed: a table is the part of this + # format meant to grow, and holding every cell as a python + # object beside the frame pandas builds would cost several + # times what the frame itself does. + _check_widths(reader, header, path) + except (OSError, UnicodeDecodeError) as read_error: + msg = f"Could not read {quote_path(path)}: {read_error}." + raise ParameterError(msg) from read_error + if not header: + msg = f"{quote_path(path)} has no columns, so it states {what}." + raise ParameterError(msg) + repeated = sorted({x for x in header if header.count(x) > 1}) + if repeated: + msg = ( + f"{quote_path(path)} names {', '.join(repeated)} more than once; one " + "column states one field." + ) + raise ParameterError(msg) + # index_col=False so that no column is ever read as an index; the row + # widths above already agree, and this keeps them agreeing. + return pd.read_csv( + path, + dtype=str, + keep_default_na=False, + na_values=[""], + index_col=False, + # Both readers decode alike, or the header checked above is not + # the header parsed here: a locale-encoded read disagrees with + # pandas' UTF-8, and a byte order mark reaches only one of them. + encoding="utf-8-sig", + ) + + +def _check_widths(reader, header: list[str], path: Path) -> None: + """ + Refuse a row which is not its header wide. + + Pandas refuses neither a wide row nor a narrow one: by default the + surplus cell pushes the first column into the index, so every value in + the row shifts one field left and lands in its neighbour's meaning. A + row states one cell per column or it is not a row. + """ + for number, row in enumerate(reader, start=2): + if row and len(row) != len(header): + msg = ( + f"{quote_path(path)} row {number} states {len(row)} cells where " + f"its header names {len(header)} columns." + ) + raise ParameterError(msg) + + +def row_cells(row) -> dict[str, str]: + """ + Return a row's stated cells, an empty one meaning unset. + + Examples + -------- + >>> import pandas as pd + >>> from dascore.utils.tables import row_cells + >>> frame = pd.DataFrame({"group": ["rail"], "value": [None]}) + >>> row_cells(frame.iloc[0]) + {'group': 'rail'} + """ + return {str(k): v for k, v in row.items() if not pd.isnull(v)} + + +def require_columns(frame: pd.DataFrame, needed, path: Path) -> None: + """ + Refuse a table which does not carry a column it is read by. + + Parameters + ---------- + frame + The table to check. + needed + The column names required; a None is ignored, so a caller may pass + an optional column straight through. + path + The file the table was read from, for the error message. + + Examples + -------- + >>> import pandas as pd + >>> from pathlib import Path + >>> from dascore.utils.tables import require_columns + >>> frame = pd.DataFrame({"sequence": [1]}) + >>> require_columns(frame, ["sequence", None], Path("t/x.csv")) is None + True + """ + missing = [x for x in needed if x is not None and x not in frame.columns] + if missing: + msg = ( + f"{quote_path(path)} states no {', '.join(missing)} column, which its " + "rows are read by." + ) + raise ParameterError(msg) + + +def require_stated(frame: pd.DataFrame, needed, path: Path) -> None: + """ + Refuse a blank cell in a column the table is read by. + + A column which orders or groups the rows decides where each one goes, + so a row leaving it empty has no place. Left to pandas the row would + simply disappear -- a null sorts last, and a null grouping key drops + its row from every group. + + Parameters + ---------- + frame + The table to check. + needed + The column names which must be stated by every row; a None is + ignored. + path + The file the table was read from, for the error message. + + Examples + -------- + >>> import pandas as pd + >>> from pathlib import Path + >>> from dascore.utils.tables import require_stated + >>> frame = pd.DataFrame({"sequence": [1, 2]}) + >>> require_stated(frame, ["sequence"], Path("t/x.csv")) is None + True + """ + for column in needed: + if column is None: + continue + empty = [ + str(n) for n, ok in enumerate(frame[column].notna(), start=2) if not ok + ] + if empty: + msg = ( + f"{quote_path(path)} leaves {column} empty at row(s) " + f"{', '.join(empty)}, so those rows state no place." + ) + raise ParameterError(msg) + + +def ordered_rows(frame: pd.DataFrame, column: str | None, path: Path) -> pd.DataFrame: + """ + Return the rows in the order the named column states, if any. + + A table which names one is read by it rather than by row position, so + re-sorting a spreadsheet cannot change what it means. A table which + names none keeps the order it was written in. + + Parameters + ---------- + frame + The table to order. + column + The numeric column stating the order, or None to keep the written + order. + path + The file the table was read from, for the error message. + + Examples + -------- + >>> import pandas as pd + >>> from pathlib import Path + >>> from dascore.utils.tables import ordered_rows + >>> frame = pd.DataFrame({"sequence": ["2", "1"], "name": ["b", "a"]}) + >>> list(ordered_rows(frame, "sequence", Path("t/x.csv"))["name"]) + ['a', 'b'] + """ + if column is None: + return frame + try: + keys = pd.to_numeric(frame[column]) + except (TypeError, ValueError) as convert_error: + msg = f"{quote_path(path)} has a non-numeric {column}: {convert_error}." + raise ParameterError(msg) from convert_error + return frame.assign(**{column: keys}).sort_values(column, kind="stable") + + +def parse_cell(text: str): + """ + Read a cell's value the way its own text states it. + + A CSV has no types, so a value which may be a string, a boolean or a + number is decided by what was written. A value which is genuinely a + string but looks like one of the others is the one thing this spelling + cannot express; that value is authored in YAML, where the types are + explicit. + + Examples + -------- + >>> from dascore.utils.tables import parse_cell + >>> parse_cell("True"), parse_cell("1e3"), parse_cell("1.5"), parse_cell("car") + (True, 1000, 1.5, 'car') + """ + if (folded := text.strip().casefold()) in ("true", "false"): + return folded == "true" + try: + number = float(text) + except ValueError: + return text + # int(number) rather than int(text): 1e3 is integral, and only the + # number knows that -- the text raises. + return int(number) if number.is_integer() and "." not in text else number diff --git a/tests/test_utils/test_intervals.py b/tests/test_utils/test_intervals.py new file mode 100644 index 000000000..188c99b1f --- /dev/null +++ b/tests/test_utils/test_intervals.py @@ -0,0 +1,171 @@ +"""Tests for half-open interval helpers.""" + +from __future__ import annotations + +import numpy as np +import pytest +from pydantic import BaseModel + +from dascore.exceptions import ParameterError +from dascore.utils.intervals import ( + clip_intervals, + interval_masks, + intervals_overlap, + normalize_value, + value_kind, +) + + +class _Span(BaseModel): + """A minimal interval item with the default field names.""" + + start_distance: float + end_distance: float + label: str = "" + + +class _Window(BaseModel): + """An interval item naming its bounds something else.""" + + time_start: float + time_end: float + + +class TestIntervalMasks: + """Coverage is half-open apart from the end of a run.""" + + def test_half_open(self): + """The start is covered and everything past the end is not.""" + (mask,) = interval_masks([-1, 0, 1, 3], [(0, 2)]) + assert list(mask) == [False, True, True, False] + + def test_run_end_included(self): + """The last value of a coverage run belongs to the interval ending there.""" + first, second = interval_masks([0, 1, 2, 3], [(0, 2), (2, 3)]) + assert list(first) == [True, True, False, False] + assert list(second) == [False, False, True, True] + + def test_claimed_end_not_shared(self): + """A value another interval already claims stays with that interval.""" + first, second = interval_masks([0, 1, 2], [(0, 2), (2, 4)]) + assert list(first) == [True, True, False] + assert list(second) == [False, False, True] + + def test_point_marker_covers_nothing(self): + """Equal start and end cover no values at all.""" + (mask,) = interval_masks([0, 1, 2], [(1, 1)]) + assert not mask.any() + + def test_returns_one_mask_per_interval(self): + """Every interval gets a mask, in the order it was given.""" + masks = interval_masks([0, 1], [(0, 1), (5, 6), (1, 1)]) + assert len(masks) == 3 + assert all(len(x) == 2 for x in masks) + + +class TestIntervalsOverlap: + """The first overlapping pair, or None.""" + + def test_touching_do_not_overlap(self): + """Half-open intervals sharing an endpoint are disjoint.""" + assert intervals_overlap([(0, 2), (2, 4)]) is None + + def test_overlap_found(self): + """An overlapping pair comes back in sorted order.""" + assert intervals_overlap([(2, 4), (0, 3)]) == ((0, 3), (2, 4)) + + def test_point_markers_ignored(self): + """A point marker covers nothing so it cannot overlap.""" + assert intervals_overlap([(0, 2), (1, 1)]) is None + + def test_empty(self): + """Nothing to compare means no overlap.""" + assert intervals_overlap([]) is None + + +class TestClipIntervals: + """Clipping keeps coverage, drops what falls outside, and keeps points.""" + + def test_clipped_to_bounds(self): + """An interval straddling the clip is trimmed to it.""" + (out,) = clip_intervals([_Span(start_distance=0, end_distance=10)], 2, 6) + assert (out.start_distance, out.end_distance) == (2, 6) + + def test_outside_dropped(self): + """An interval left with no coverage is dropped.""" + assert clip_intervals([_Span(start_distance=8, end_distance=10)], 2, 6) == [] + + def test_other_fields_kept(self): + """Only the bounds change; the item itself is left alone.""" + span = _Span(start_distance=0, end_distance=10, label="rail") + (out,) = clip_intervals([span], 2, 6) + assert out.label == "rail" + assert (span.start_distance, span.end_distance) == (0, 10) + + def test_point_inside_survives(self): + """A point marker inside the clip is kept as it was.""" + point = _Span(start_distance=3, end_distance=3) + assert clip_intervals([point], 2, 6) == [point] + + def test_point_on_outer_end_survives(self): + """A point on the outermost included endpoint is not lost.""" + point = _Span(start_distance=6, end_distance=6) + assert clip_intervals([point], 2, 6) == [] + assert clip_intervals([point], 2, 6, outer=6) == [point] + + def test_field_names(self): + """Any pair of start/end fields works, not just the inventory's.""" + window = _Window(time_start=0, time_end=10) + (out,) = clip_intervals( + [window], 2, 6, start_field="time_start", end_field="time_end" + ) + assert (out.time_start, out.time_end) == (2, 6) + + +class TestValueKind: + """The kind decides the shape of the group a value belongs to.""" + + def test_bool_before_int(self): + """A bool is a boolean even though it is also an int.""" + assert value_kind(True) == "boolean" + + def test_string(self): + """Text is a string kind.""" + assert value_kind("car") == "string" + + @pytest.mark.parametrize("value", [1, 1.5, -3]) + def test_numeric(self, value): + """Ints and floats share one kind.""" + assert value_kind(value) == "numeric" + + +class TestNormalizeValue: + """Values keep their python type and must be finite.""" + + def test_numpy_bool_unwrapped(self): + """A numpy bool becomes a python bool, not a number.""" + assert normalize_value(np.bool_(True)) is True + + def test_numpy_int_unwrapped(self): + """A numpy int becomes a python int, not a float.""" + out = normalize_value(np.int64(5)) + assert isinstance(out, int) and not isinstance(out, bool) + + def test_python_value_untouched(self): + """A plain value comes back as it went in.""" + assert normalize_value("car") == "car" + + @pytest.mark.parametrize("value", [np.nan, np.inf, -np.inf]) + def test_non_finite_refused(self, value): + """A non-finite number cannot survive a JSON round trip.""" + with pytest.raises(ParameterError, match="must be finite"): + normalize_value(value) + + def test_error_class(self): + """The caller's format decides which exception it raises.""" + + class _MyError(ValueError): + """A format-specific error.""" + + with pytest.raises(_MyError, match="must be finite"): + normalize_value(np.nan, error=_MyError) diff --git a/tests/test_utils/test_tables.py b/tests/test_utils/test_tables.py new file mode 100644 index 000000000..4852b5133 --- /dev/null +++ b/tests/test_utils/test_tables.py @@ -0,0 +1,190 @@ +"""Tests for the strict CSV table reader.""" + +from __future__ import annotations + +import pandas as pd +import pytest + +from dascore.exceptions import ParameterError +from dascore.utils.tables import ( + ordered_rows, + parse_cell, + read_table, + require_columns, + require_stated, + row_cells, +) + + +def _write(path, text: str, name: str = "table.csv", encoding: str = "utf-8"): + """Write a table and return its path.""" + out = path / name + out.write_text(text, encoding=encoding) + return out + + +class TestReadTable: + """Reading is strict about shape and indifferent to pandas' inference.""" + + def test_every_cell_is_text(self, tmp_path): + """A column's type is the caller's business, not pandas' guess.""" + path = _write(tmp_path, "sequence,value\n1,2.0\n") + frame = read_table(path) + assert list(frame["sequence"]) == ["1"] + assert list(frame["value"]) == ["2.0"] + + def test_only_empty_is_null(self, tmp_path): + """A cell saying NA means the string; a blank one means unset.""" + path = _write(tmp_path, "a,b\nNA,\n") + frame = read_table(path) + assert frame["a"][0] == "NA" + assert pd.isnull(frame["b"][0]) + + def test_byte_order_mark(self, tmp_path): + """A mark written by a spreadsheet is not part of the first header.""" + path = _write(tmp_path, "a,b\n1,2\n", encoding="utf-8-sig") + assert list(read_table(path).columns) == ["a", "b"] + + def test_no_columns_refused(self, tmp_path): + """An empty file states nothing.""" + path = _write(tmp_path, "") + with pytest.raises(ParameterError, match="no columns, so it states nothing"): + read_table(path) + + def test_no_columns_names_what_is_missing(self, tmp_path): + """The caller's format names what a column-less file fails to state.""" + path = _write(tmp_path, "") + with pytest.raises(ParameterError, match="states no track"): + read_table(path, what="no track") + + def test_repeated_column_refused(self, tmp_path): + """One column states one field; pandas would silently rename it.""" + path = _write(tmp_path, "a,a\n1,2\n") + with pytest.raises(ParameterError, match="more than once"): + read_table(path) + + def test_wide_row_refused(self, tmp_path): + """A surplus cell would shift every value one field left.""" + path = _write(tmp_path, "a,b\n1,2,3\n") + with pytest.raises(ParameterError, match="3 cells"): + read_table(path) + + def test_narrow_row_refused(self, tmp_path): + """A missing cell is refused rather than filled in.""" + path = _write(tmp_path, "a,b\n1\n") + with pytest.raises(ParameterError, match="1 cells"): + read_table(path) + + def test_missing_file(self, tmp_path): + """An unreadable file raises the caller's error, not OSError.""" + with pytest.raises(ParameterError, match="Could not read"): + read_table(tmp_path / "absent.csv") + + +class TestRowCells: + """Only stated cells are reported.""" + + def test_unset_dropped(self): + """A blank cell means unset rather than a null value.""" + frame = pd.DataFrame({"a": ["x"], "b": [None]}) + assert row_cells(frame.iloc[0]) == {"a": "x"} + + +class TestRequireColumns: + """A table must carry the columns it is read by.""" + + def test_present(self, tmp_path): + """Nothing happens when every needed column is there.""" + frame = pd.DataFrame({"sequence": ["1"]}) + assert require_columns(frame, ["sequence"], tmp_path / "t.csv") is None + + def test_none_ignored(self, tmp_path): + """A None names no column, so it is not missing.""" + frame = pd.DataFrame({"sequence": ["1"]}) + assert require_columns(frame, [None], tmp_path / "t.csv") is None + + def test_missing_refused(self, tmp_path): + """A missing column is named in the error.""" + frame = pd.DataFrame({"a": ["1"]}) + with pytest.raises(ParameterError, match="no sequence column"): + require_columns(frame, ["sequence"], tmp_path / "t.csv") + + +class TestRequireStated: + """A row leaving an ordering or grouping cell blank has no place.""" + + def test_stated(self, tmp_path): + """Nothing happens when every row states the column.""" + frame = pd.DataFrame({"sequence": ["1", "2"]}) + assert require_stated(frame, ["sequence"], tmp_path / "t.csv") is None + + def test_none_ignored(self, tmp_path): + """A None names no column, so nothing is required.""" + frame = pd.DataFrame({"sequence": ["1"]}) + assert require_stated(frame, [None], tmp_path / "t.csv") is None + + def test_blank_refused(self, tmp_path): + """The refused row is named by its line in the file.""" + frame = pd.DataFrame({"sequence": ["1", None, "3"]}) + with pytest.raises(ParameterError, match="row\\(s\\) 3"): + require_stated(frame, ["sequence"], tmp_path / "t.csv") + + +class TestOrderedRows: + """Rows are read in the order their column states.""" + + def test_sorted_numerically(self, tmp_path): + """Text digits order as numbers, not as strings.""" + frame = pd.DataFrame({"sequence": ["10", "2"], "name": ["b", "a"]}) + out = ordered_rows(frame, "sequence", tmp_path / "t.csv") + assert list(out["name"]) == ["a", "b"] + + def test_stable(self, tmp_path): + """Rows sharing a key keep the order they were written in.""" + frame = pd.DataFrame({"sequence": ["1", "1"], "name": ["b", "a"]}) + out = ordered_rows(frame, "sequence", tmp_path / "t.csv") + assert list(out["name"]) == ["b", "a"] + + def test_no_column_keeps_order(self, tmp_path): + """A table naming no ordering column is left as written.""" + frame = pd.DataFrame({"name": ["b", "a"]}) + out = ordered_rows(frame, None, tmp_path / "t.csv") + assert list(out["name"]) == ["b", "a"] + + def test_non_numeric_refused(self, tmp_path): + """An ordering column which is not numeric orders nothing.""" + frame = pd.DataFrame({"sequence": ["first"]}) + with pytest.raises(ParameterError, match="non-numeric sequence"): + ordered_rows(frame, "sequence", tmp_path / "t.csv") + + +class TestParseCell: + """A cell's value is decided by what the cell says.""" + + @pytest.mark.parametrize("text", ["true", "True", " TRUE "]) + def test_true(self, text): + """Booleans are read regardless of case or padding.""" + assert parse_cell(text) is True + + def test_false(self): + """False is a boolean, not the string.""" + assert parse_cell("false") is False + + def test_integer(self): + """A whole number is an int.""" + out = parse_cell("5") + assert out == 5 and isinstance(out, int) + + def test_exponent_is_integral(self): + """1e3 is integral even though its text is not.""" + out = parse_cell("1e3") + assert out == 1000 and isinstance(out, int) + + def test_float_keeps_its_point(self): + """A number written with a point stays a float.""" + out = parse_cell("2.0") + assert out == 2.0 and isinstance(out, float) + + def test_text(self): + """Anything else is the string it was written as.""" + assert parse_cell("car") == "car"