diff --git a/dascore/core/annotation_loader.py b/dascore/core/annotation_loader.py index a7dd255fd..f7e276863 100644 --- a/dascore/core/annotation_loader.py +++ b/dascore/core/annotation_loader.py @@ -6,6 +6,19 @@ own dimensions and provenance; ``vertices.csv`` where any path or polygon needs one -- or as a bare table whose dimensions the caller states. +A directory of those directories is a collection, and reads as one set +whose ``set`` column names which of them each row came from: one return +type, so nothing downstream has to ask which layout it was handed. What a +set declares only for itself -- its dimensions, its provenance, its +documented columns -- is kept under ``attrs.sets``, and a row's identity is +still the ``id`` it already had, so an id two sets share is refused rather +than qualified by the set it came from. + +A table may declare the dimensions it is stated in itself, in a +``# dims: distance, time`` comment above its header, and a directory of data +carries the annotations made on it under the hidden name ``.annotations``, as +it carries its inventory under ``.inventory``. + CSV has no types, so this module decides what each column holds before the models see it: a dimension column is numbers or times, a ``basis`` cell is the JSON document its curve dumps, and every other cell is read the way it @@ -20,8 +33,8 @@ import json import math import os -from collections.abc import Mapping, Sequence -from contextlib import suppress +from collections.abc import Collection, Mapping, Sequence +from contextlib import contextmanager, suppress from pathlib import Path from typing import Any @@ -35,17 +48,26 @@ _VERTEX_COLUMNS, ANNOTATION_STEM, ATTRS_STEM, + DIMS_KEY, OBJECT_SUFFIXES, RESERVED_COLUMNS, - TABLE_SUFFIX, + TABLE_SUFFIXES, VERTEX_STEM, AnnotationSet, + _text, + annotation_set_to_dataframe, + annotation_set_to_vertices, ) from dascore.exceptions import InvalidAnnotationError, ParameterError from dascore.models.registry import TAG_FIELD from dascore.utils.misc import iterate, optional_import from dascore.utils.paths import quote_path -from dascore.utils.tables import parse_cell, read_table +from dascore.utils.tables import ( + parse_cell, + read_parquet, + read_parquet_metadata, + read_table, +) from dascore.utils.time import to_datetime64 # What an attrs file declares itself to be; the model writes its own tag. @@ -61,6 +83,22 @@ # The column a vertex states its place in the order by; a number. _ORDINAL = _VERTEX_COLUMNS[1] +# What a table may carry above its header: comment lines, one of which may +# declare the dimensions the table is stated in. +_COMMENT = "#" +_DIMS_PRAGMA = "dims" + +# The suffix which names the parquet encoding, for the branches which read +# a table rather than merely find one. +PARQUET_SUFFIX = TABLE_SUFFIXES[1] + +# The name a directory of data carries its annotations under, in either +# form: the directory `.annotations/`, holding a set or a directory of sets, +# or the bare table `.annotations.csv`. Hidden, like the `.inventory` which +# may sit beside it -- a companion the directory keeps rather than content it +# holds, and what keeps the file scanner from reading it as data. +BLESSED_NAME = ".annotations" + def _read_object(path: Path) -> dict[str, Any]: """Parse one YAML or JSON object file into a mapping.""" @@ -93,11 +131,27 @@ def _read_object(path: Path) -> dict[str, Any]: return dict(data) +def _entries(directory: Path) -> list[Path]: + """ + Return what a directory holds, as this format's own error. + + ``iterdir`` and the ``exists`` calls in the scans raise ``OSError`` -- a + directory whose permissions were tightened is the ordinary case -- and + that would leave `annotations` unwrapped, where every other failure to + read a stored set arrives as an annotation error. + """ + try: + return sorted(directory.iterdir()) + except OSError as error: + msg = f"Could not read {quote_path(directory)}: {error}." + raise ParameterError(msg) from error + + def _one_spelling(directory: Path, stem: str, suffixes: Sequence[str]) -> Path | None: """Return the one file a stem names, or None; two spellings raise.""" found = [ x - for x in sorted(directory.iterdir()) + for x in _entries(directory) if x.stem == stem and x.suffix.casefold() in suffixes ] if len(found) > 1: @@ -189,8 +243,17 @@ def _dimension_spellings(dims: Sequence[str]) -> frozenset[str]: return frozenset(x for dim in dims for x in (dim, f"{dim}{_START}", f"{dim}{_END}")) +def _is_text(series: pd.Series) -> bool: + """Whether a column holds text, whichever way pandas is spelling it.""" + return getattr(series.dtype, "kind", "") in "OTU" + + def _read_cells( - frame: pd.DataFrame, dims: Sequence[str], path: Path, ordered: bool = False + frame: pd.DataFrame, + dims: Sequence[str], + path: Path, + ordered: bool = False, + typed: bool = False, ) -> pd.DataFrame: """ Read a table's text cells as the values each column holds. @@ -198,17 +261,37 @@ def _read_cells( Only the vertices are ordered, so only they read ``seq`` as the number it is: the annotations table does not reserve that name, and an annotation carrying its own ``seq`` means whatever it says. + + A typed table -- parquet -- states what each column holds, so only its + text columns are read further, and only as far as a dimension or a + basis: text elsewhere is text, since a format with a boolean of its own + would have used one. Stating a type is not the same as stating a usable + one, so a dimension or a vertex order which arrives as something a + coordinate cannot be is refused rather than trusted. """ spellings = _dimension_spellings(dims) out = {} for name in frame.columns: series = frame[name] - if str(name) in spellings: + if typed and not _is_text(series): + # The file stated what this column holds, so nothing here has + # to work it out from a spelling -- only check that what it + # states is a thing the column is allowed to hold. + if str(name) in spellings: + _check_kind(series, name, path, "iufMm", "numbers or times") + elif ordered and str(name) == _ORDINAL: + _check_kind(series, name, path, "iuf", "a number") + out[name] = series + elif str(name) in spellings: out[name] = _read_dimension(series, path) elif ordered and str(name) == _ORDINAL: out[name] = _read_ordinal(series, path) elif str(name) == "basis": out[name] = _read_basis(series, path) + elif typed: + # Text in a typed table is text: a cell reading 'true' in a + # format which has a boolean is the word, not the boolean. + out[name] = series elif str(name) in _TEXT_COLUMNS or series.isna().all(): # A column no row states says nothing about what it holds, and # reading its emptiness as a type would invent one. @@ -218,6 +301,17 @@ def _read_cells( return pd.DataFrame(out) +def _check_kind(series: pd.Series, name, path: Path, kinds: str, what: str) -> None: + """Refuse a typed column whose type the field it names cannot be.""" + if series.dtype.kind in kinds: + return + msg = ( + f"The column {str(name)!r} of {quote_path(path)} holds " + f"{series.dtype}, where it states {what}." + ) + raise ParameterError(msg) + + def _read_extra(cell): """ Read one cell of a column the set does not model. @@ -235,46 +329,175 @@ def _read_extra(cell): def _read_set_table( - path: Path, dims: Sequence[str], what: str, ordered: bool = False + path: Path, + dims: Sequence[str], + what: str, + ordered: bool = False, + skip: int = 0, ) -> pd.DataFrame | None: """ Read one of a set's tables, with its cells typed. - A table stating nothing at all is what a set of no annotations writes, - so it reads back as none rather than as a table with no columns. + A table stating nothing at all is what a set of no annotations writes + -- a blank CSV, or a parquet file with no columns -- so it reads back + as none rather than as a table which states nothing. """ + if _is_parquet(path): + frame, _ = read_parquet(path, what=what, empty=True) + if not len(frame.columns): + return None + return _read_cells(frame, dims, path, ordered=ordered, typed=True) if _is_blank(path): return None - return _read_cells(read_table(path, what=what), dims, path, ordered=ordered) + frame = read_table(path, what=what, skip=skip) + return _read_cells(frame, dims, path, ordered=ordered) + + +def _is_parquet(path: Path) -> bool: + """Whether a table's name says it is parquet rather than CSV.""" + return path.suffix.casefold() == PARQUET_SUFFIX + + +def _read_table_dims(path: Path) -> tuple[tuple[str, ...] | None, int]: + """ + Return the dimensions a table declares for itself, and the CSV lines + above its header. + + Each encoding declares them where it can: a CSV in a comment above the + header, a parquet file in the metadata its footer holds. + """ + if not _is_parquet(path): + return _read_pragma(path) + stated = read_parquet_metadata(path) + return _stated_dims(stated.get(DIMS_KEY), path), 0 + + +def _stated_dims(document: str | None, path: Path) -> tuple[str, ...] | None: + """Read the dimensions a parquet file states in its metadata.""" + if document is None: + return None + try: + stated = json.loads(document) + except ValueError as error: + msg = ( + f"{quote_path(path)} states {DIMS_KEY} as {document!r}, which is not " + f"a JSON document: {error}." + ) + raise ParameterError(msg) from error + names = tuple(str(x) for x in iterate(stated)) + if not names: + msg = ( + f"{quote_path(path)} states {DIMS_KEY} but names none; a table which " + "declares its dimensions names them." + ) + raise ParameterError(msg) + return names + + +def _read_pragma(path: Path) -> tuple[tuple[str, ...] | None, int]: + """ + Return the dimensions a table declares above its header, and the lines + to skip to reach that header. + + A table which states its dimensions nowhere else may declare them in a + comment above its header:: + + # dims: distance, time + # picked by hand + group,time_start,time_end + + Nothing above the header is skipped unless one of those lines is the + declaration. A column name may begin with the comment mark -- `# note` + is a name a set can hold and this library writes unquoted -- and eating + that header would promote the first row of data to the header with + nothing said. Where a table does declare its dimensions its author has + opted into the convention, and further comments beside the declaration + are skipped with it. + + A comment because a CSV has nowhere else to put this. A reader told + ``comment="#"`` skips the line; one which is not told reads it as the + header, which is the cost of the convention and the reason `to_csv` + does not write it. + """ + stated: tuple[str, ...] | None = None + skip = 0 + with _readable(path) as stream: + for line in stream: + bare = line.strip() + if bare and not bare.startswith(_COMMENT): + break + skip += 1 + name, _, rest = bare.removeprefix(_COMMENT).partition(":") + if name.strip().casefold() != _DIMS_PRAGMA: + continue + if stated is not None: + msg = ( + f"{quote_path(path)} declares {_DIMS_PRAGMA} more than once " + "above its header; a table states its dimensions once." + ) + raise ParameterError(msg) + stated = tuple(x.strip() for x in rest.split(",") if x.strip()) + if not stated: + msg = ( + f"{quote_path(path)} declares {_DIMS_PRAGMA} above its " + "header but names none; write '# dims: distance, time'." + ) + raise ParameterError(msg) + # No declaration, so there was no preamble to skip: whatever those lines + # were, the first of them is this table's header. + return (stated, skip) if stated is not None else (None, 0) + + +@contextmanager +def _readable(path: Path): + """ + Open a table for the scan above its header, naming what cannot be read. + + Swallowing the failure would be worse than reporting it: a table which + does not decode has no dimensions to find, and the caller would go on to + advise writing the very line the file already holds. + """ + try: + with path.open(encoding="utf-8-sig") as stream: + yield stream + except (OSError, UnicodeDecodeError) as error: + msg = f"Could not read {quote_path(path)}: {error}." + raise ParameterError(msg) from error def _is_blank(path: Path) -> bool: """Whether a table holds nothing but whitespace.""" - try: - return not path.read_text(encoding="utf-8-sig").strip() - except (OSError, UnicodeDecodeError): - # Unreadable is the table reader's to name, with its own message. - return False + with _readable(path) as stream: + return not stream.read().strip() + + +def _tables(directory: Path) -> list[Path]: + """ + Every file in a directory whose name says it is a table. + + Hidden names are not tables here, as they are not sets: an editor's + ``.annotations.csv.swp`` or a half-copied ``.annotations.csv`` is a + companion the directory keeps, and taking a directory down for one would + make a stray sync file fatal. + """ + return [ + x + for x in _entries(directory) + if not x.name.startswith(".") and x.suffix.casefold() in TABLE_SUFFIXES + ] -def _refuse_stray_tables(directory: Path) -> None: +def _refuse_stray_tables(directory: Path, known: Collection[str], what: str) -> None: """ - Refuse a table whose name names no part of a set. + Refuse a table whose name names no part of what a directory holds. A ``vertexes.csv`` beside an ``annotations.csv`` claims to participate in this convention and gets it wrong, which is worth more than being quietly skipped. """ - known = {ANNOTATION_STEM, VERTEX_STEM} - stray = sorted( - x.name for x in directory.glob(f"*{TABLE_SUFFIX}") if x.stem not in known - ) + stray = sorted(x.name for x in _tables(directory) if x.stem not in known) if stray: - msg = ( - f"{quote_path(directory)} holds the table(s) {', '.join(stray)}, which " - f"name no part of a set. A set states {ANNOTATION_STEM}{TABLE_SUFFIX} " - f"and, where it has vertices, {VERTEX_STEM}{TABLE_SUFFIX}." - ) + msg = f"{quote_path(directory)} holds the table(s) {', '.join(stray)}, {what}" raise ParameterError(msg) @@ -297,34 +520,502 @@ def _refuse_overrides(what: str, **stated) -> None: def _load_directory(directory: Path, dims, **kwargs) -> AnnotationSet: - """Load the set a directory holds.""" - # Both are the directory's to state, and passing them through would - # reach AnnotationSet twice as a bare TypeError. + """Load the set a directory holds, or the sets a directory of them does.""" + # A directory which states annotations is a set, and nothing below it is + # looked at: a folder someone kept beside its tables -- a backup of an + # attrs file, an older copy of the set -- is no more this format's + # business than a notes.txt is, and reading the directory as a + # collection because of one would refuse a set which is complete. + children: list[Path] = [] + if not _states_annotations(directory): + children = _child_sets(directory) + # A directory which states nothing itself may still carry + # annotations, under the hidden name, as a directory of data carries + # its inventory. Nothing of its own is read on this path: a data + # directory's attrs.json is about the data, and what the caller + # states is the carried table's to take, since it states none of it. + if not children and (carried := find_annotations(directory)) is not None: + return _load_path(carried, dims, **kwargs) + _refuse_stated(directory, kwargs) + attrs = _read_attrs(directory) + if children: + return _load_collection(directory, children, attrs, dims, **kwargs) + return _load_set(directory, attrs, dims, **kwargs) + + +def _given_attrs(kwargs: Mapping) -> Mapping: + """Return what a caller stated for a source which states nothing itself.""" + attrs = kwargs.get("attrs") + if attrs is None: + return {} + return attrs if isinstance(attrs, Mapping) else attrs.model_dump() + + +def _refuse_stated(directory: Path, kwargs: dict) -> None: + """ + Refuse attributes or vertices given for a directory which states them. + + Refused for a directory which holds a set or the sets, not for one + carrying a bare `.annotations.csv`: that table states neither, so a + caller has the same say over what it holds as it has passing the table + itself. Consumed rather than merely refused, since what a directory + states would otherwise reach `AnnotationSet` twice as a bare TypeError. + """ _refuse_overrides( - "a set directory", + f"{quote_path(directory)}, which states them", attrs=kwargs.pop("attrs", None), vertices=kwargs.pop("vertices", None), ) - attrs = _read_attrs(directory) - # Dimensions too, once the directory has stated them: reading the + + +def _load_path(path: Path, dims, **kwargs) -> AnnotationSet: + """Load whichever of the two shapes a path holds.""" + if path.is_dir(): + return _load_directory(path, dims, **kwargs) + return _load_file(path, dims, **kwargs) + + +def _states_annotations(path: Path) -> bool: + """ + Whether a directory states annotations of its own, in either encoding. + + A file states none: the scans below walk what a directory holds, and a + plain file among them is not a set which spelled itself oddly. + """ + if not path.is_dir(): + return False + return _one_spelling(path, ANNOTATION_STEM, TABLE_SUFFIXES) is not None + + +def find_annotations(directory: str | os.PathLike) -> Path | None: + """ + Return what a directory of data carries its annotations under, or None. + + Only the name is judged, never the contents, as + [find_inventory](`dascore.core.inventory_loader.find_inventory`) judges + the inventory's: a hidden ``.annotations/`` holds the set, or the sets, + a directory of data was annotated with, and ``.annotations.csv`` is the + bare-table spelling of the same thing. Hidden, so the file scanner does + not read it as data, and so a directory which states a visible + ``annotations.csv`` is a set rather than something carrying one. + + Raises rather than answering where a directory says two things at once + -- both spellings present -- or where what sits under the name is the + wrong kind of thing for it, which is a misspelling of the convention + rather than a file which owes it nothing. `InvalidAnnotationError`, as + `find_inventory` raises the inventory's own: this is a door callers use + directly, so it fails the way the rest of `dc.annotations` fails. + + Parameters + ---------- + directory + The directory to look in. + """ + root = Path(directory) + tree = root / BLESSED_NAME + named = ", ".join(tree.with_suffix(x).name for x in TABLE_SUFFIXES) + # The suffix is matched without regard to case, as every other table + # this format finds is: a `.annotations.PARQUET` is the file the lower + # case name would be, and reading one and not the other would make the + # carried name mean less than the visible one. + tables = [ + x + for x in _entries(root) + if x.stem == BLESSED_NAME and x.suffix.casefold() in TABLE_SUFFIXES + ] + found = [x for x in (tree, *tables) if x.exists()] + if not found: + return None + if len(found) > 1: + listed = ", ".join(x.name for x in found) + msg = ( + f"{quote_path(root)} carries annotations more than once: {listed}. " + "A directory states what it carries once; keep the one it means." + ) + raise InvalidAnnotationError(msg) + (only,) = found + if only == tree and not only.is_dir(): + msg = ( + f"{quote_path(only)} is a file. The annotations a directory carries " + f"are the set directory {BLESSED_NAME}/, or a bare table: {named}." + ) + raise InvalidAnnotationError(msg) + if only != tree and only.is_dir(): + msg = ( + f"{quote_path(only)} is a directory. A set held as a directory is " + f"named {BLESSED_NAME}/; a name with a suffix spells a bare table." + ) + raise InvalidAnnotationError(msg) + return only + + +def _child_sets(directory: Path) -> list[Path]: + """ + Return the set directories a directory of sets holds, in name order. + + Only reached for a directory which states no annotations itself. The + sets are found first and nothing is refused until at least one is: a + directory holding none of them is not a collection at all -- it is the + data, or a directory carrying its annotations under the hidden name -- + and complaining about what sits in it would refuse a directory this + format has no claim on. + + Once it is a collection, a child which states attributes and no + annotations is half a set and says so, and one holding sets of its own + is refused, since sets loaded together are one collection rather than a + tree. Anything else is left alone -- the data the sets describe, a + folder of figures -- as is a directory holding only vertices, which is + never looked for without the annotations it belongs to. + """ + # Hidden names are skipped as the file scanner skips them: a + # `.inventory` beside the sets describes the data, not the annotations. + children = [ + x for x in _entries(directory) if x.is_dir() and not x.name.startswith(".") + ] + out = [x for x in children if _states_annotations(x)] + if not out: + return [] + for child in (x for x in children if x not in out): + if nested := [x.name for x in _entries(child) if _states_annotations(x)]: + msg = ( + f"{quote_path(child)} holds the set(s) {', '.join(nested)}. Sets " + "loaded together are one collection, not a tree of them." + ) + raise ParameterError(msg) + if _one_spelling(child, ATTRS_STEM, OBJECT_SUFFIXES) is not None: + msg = ( + f"{quote_path(child)} states the attributes of a set but no " + f"{ANNOTATION_STEM} table, so it states no annotations." + ) + raise ParameterError(msg) + _refuse_colliding_names(directory, out) + return out + + +def _refuse_colliding_names(directory: Path, children: Sequence[Path]) -> None: + """ + Refuse set names which differ only in case. + + The name is the label: it is the `set` column and the key in + ``attrs.sets``, so two which fold together are one set on a filesystem + which folds them and two on this one. A symlinked set is allowed, by + contrast -- a collection is a convenience for reading, not an authored + identity, so pointing one at a set kept elsewhere is a fair use of it. + """ + folded: dict[str, str] = {} + for child in children: + first = folded.setdefault(child.name.casefold(), child.name) + if first != child.name: + msg = ( + f"{quote_path(directory)} holds the sets {first} and {child.name}, " + "whose names differ only in case. A set name is the label its " + "rows carry, so it must name one set on any filesystem." + ) + raise ParameterError(msg) + + +def _load_collection( + directory: Path, children: Sequence[Path], attrs: Mapping, dims, **kwargs +) -> AnnotationSet: + """ + Load the sets a directory of them holds, as one set. + + Each child is read in its own dimensions, then merged by `_merge_sets`. + The refusals here are the ones only a collection can hit: sets stated + twice, a table beside them, and dimensions given for a set which + declares its own. + """ + if attrs.get("sets"): + msg = ( + f"{quote_path(directory)} states sets in its attributes and holds " + "them in directories. A collection states each of its sets once." + ) + raise ParameterError(msg) + _refuse_stray_tables( + directory, + (), + "which name no set. A set is a directory here, so a table beside them " + "states nothing; a bare table is read on its own.", + ) + if attrs.get("dims"): + _refuse_overrides("a directory of sets stating its own dimensions", dims=dims) + # The caller's dimensions, else the ones stated beside the sets, stand in + # for a child which declares none. A child which declares its own -- in + # its attributes or above its table -- is read in those, and refuses the + # standing-in dimensions as it would refuse them on its own, rather than + # having them dropped where nobody can see it happen. + default = dims if dims is not None else attrs.get("dims") + given = "dims" if dims is not None else "the dimensions stated beside the sets" + loaded = {} + for child in children: + child_attrs = _read_attrs(child) + if _declares_dims(child, child_attrs): + if default is not None: + msg = ( + f"{given} was given for {quote_path(child)}, which states its " + "own. Read it and change it, rather than reading it as " + "something else." + ) + raise ParameterError(msg) + loaded[child.name] = _load_set(child, child_attrs, None) + continue + loaded[child.name] = _load_set(child, child_attrs, default) + return _merge_sets(loaded, attrs, **kwargs) + + +def _declares_dims(directory: Path, attrs: Mapping) -> bool: + """ + Whether a stored set states its own dimensions, however it states them. + + A set declares them in its attributes or above its table, and the two + spellings mean the same thing here: either way the set has said what it + is read in, so the collection's own dimensions are not its to take. + """ + if attrs.get("dims"): + return True + table = _one_spelling(directory, ANNOTATION_STEM, TABLE_SUFFIXES) + return table is not None and _read_table_dims(table)[0] is not None + + +def _merge_sets( + loaded: Mapping[str, AnnotationSet], attrs: Mapping, **kwargs +) -> AnnotationSet: + """Build the one set the sets loaded together make.""" + stated = [str(x) for x in iterate(attrs.get("dims") or ())] + # The collection's own dimensions first, then each set's in the order + # the sets were read, which is their names' -- so the order is stable + # for one directory, though renaming a set can change it. + dims = tuple( + dict.fromkeys([*stated, *(x for one in loaded.values() for x in one.dims)]) + ) + frames = {name: _labeled(one, name) for name, one in loaded.items()} + _refuse_undeclared_dims(loaded, frames, dims) + _refuse_mixed_spellings(frames, dims) + _refuse_mixed_kinds(frames, dims, "an annotation") + frame = pd.concat(frames.values(), ignore_index=True, sort=False) + _refuse_shared_ids(frame) + vertices = { + name: drawn + for name, one in loaded.items() + if not (drawn := annotation_set_to_vertices(one)).empty + } + _refuse_mixed_vertices(vertices) + _refuse_mixed_kinds(vertices, dims, "a vertex") + document = dict(attrs) + document["dims"] = dims + document["sets"] = {name: one.attrs for name, one in loaded.items()} + return AnnotationSet( + frame, + vertices=pd.concat(vertices.values(), ignore_index=True, sort=False) + if vertices + else None, + attrs=document, + **kwargs, + ) + + +def _labeled(one: AnnotationSet, name: str) -> pd.DataFrame: + """ + Return one set's annotations, saying which set each row came from. + + Only the label is added. What the set states for itself stays in its + attributes, where `attrs.sets` keeps it and a row reads it back through + the label -- writing any of it into every row would store one fact + twice, in two places which can then disagree. + """ + frame = annotation_set_to_dataframe(one) + if "set" in frame.columns: + msg = ( + f"The set {name!r} states a set column, so it is already a " + "collection -- sets saved flat, most likely. A collection is not a " + "member of another one; read it on its own, or spread it back out." + ) + raise ParameterError(msg) + frame["set"] = name + return frame + + +def _refuse_undeclared_dims( + loaded: Mapping[str, AnnotationSet], frames: Mapping[str, pd.DataFrame], dims +) -> None: + """ + Refuse a column which is a dimension in one set and something else in + another. + + The collection's dimensions are the union of its sets', so a set which + holds a column another set declares as a dimension would have that + column read as a coordinate it never claimed: its extra would stop + being an extra and start bounding a region. + """ + for name, frame in frames.items(): + undeclared = set(dims) - set(loaded[name].dims) + for dim in sorted(undeclared): + spelled = [dim, f"{dim}{_START}", f"{dim}{_END}"] + held = sorted(x for x in spelled if x in frame.columns) + if not held: + continue + claims = sorted(k for k, v in loaded.items() if dim in v.dims) + msg = ( + f"The set {name} holds {', '.join(held)} without declaring " + f"{dim!r} a dimension, and {', '.join(claims)} declares it one. " + "One column states one thing, so the sets cannot be read " + "together until they agree on what it is." + ) + raise ParameterError(msg) + + +def _refuse_mixed_spellings(frames: Mapping[str, pd.DataFrame], dims) -> None: + """ + Refuse a dimension two sets spell differently. + + A set holds one spelling of a dimension, so a merged table cannot hold + both a bare ``time`` and a ``time_start``/``time_end`` pair; the + constructor refuses that too, but without naming the sets. Neither + spelling stands in for the other: a half-open range of no width holds + nothing, so a point is not a range and cannot be rewritten as one. + """ + for dim in dims: + points = [name for name, x in frames.items() if dim in x.columns] + ranges = [name for name, x in frames.items() if f"{dim}{_START}" in x.columns] + if points and ranges: + msg = ( + f"The dimension {dim!r} is spelled as a point in " + f"{', '.join(points)} and as a range in {', '.join(ranges)}, so " + "the sets state one thing two ways and cannot be read together." + ) + raise ParameterError(msg) + + +def _refuse_mixed_vertices(vertices: Mapping[str, pd.DataFrame]) -> None: + """ + Refuse sets which draw their vertices in different dimensions. + + A vertex states every dimension its table names, so a curve drawn in + distance and time cannot sit in one table beside one drawn in distance + alone: the second would leave a column empty and be half a shape. The + annotations merge because a bound may be unstated; a vertex may not. + """ + spelled = { + name: tuple(sorted(set(map(str, frame.columns)) - set(_VERTEX_COLUMNS))) + for name, frame in vertices.items() + } + if len(set(spelled.values())) < 2: + return + listed = "; ".join(f"{name} in {', '.join(dims)}" for name, dims in spelled.items()) + msg = ( + f"The sets loaded together draw their vertices in different dimensions " + f"({listed}). A vertex states every dimension its table names, so these " + "cannot be read as one table." + ) + raise ParameterError(msg) + + +def _refuse_mixed_kinds(frames: Mapping[str, pd.DataFrame], dims, what: str) -> None: + """ + Refuse a dimension two sets state in different kinds of value. + + Each set is read in its own dimensions, so one may state ``time`` as + seconds and another as dates; concatenating those gives a column of + both, which every later comparison -- a bound against a bound, a sort, + a select -- either refuses far from here or gets wrong quietly. The + kinds are compared rather than the dtypes: a column of whole numbers + and one of floats say the same kind of thing. + """ + kinds: dict[str, dict[str, str]] = {} + for name, frame in frames.items(): + for dim in dims: + for column in (dim, f"{dim}{_START}", f"{dim}{_END}"): + if column not in frame.columns: + continue + kind = _kind(frame[column]) + seen = kinds.setdefault(column, {}) + if kind != "nothing" and seen and kind not in seen: + stated, first = next(iter(seen.items())) + msg = ( + f"The sets state {what}'s {column} in different kinds of " + f"value: {kind} in {name}, {stated} in {first}. One column " + "holds one kind, so the sets cannot be read together." + ) + raise ParameterError(msg) + if kind != "nothing": + seen.setdefault(kind, name) + + +def _kind(series: pd.Series) -> str: + """Name the kind of value a column holds, as a reader would say it.""" + kind = getattr(series.dtype, "kind", "O") + if series.isna().all(): + # A column no row states says nothing about its kind, so it agrees + # with whatever the other sets state. + return "nothing" + return {"M": "times", "m": "durations", "O": "text", "T": "text", "U": "text"}.get( + kind, "numbers" + ) + + +def _refuse_shared_ids(frame: pd.DataFrame) -> None: + """ + Refuse an id which names a row in more than one set. + + Each set was built before this, so its own ids are already unique; + what is left is a collision between sets, which would make the id no + longer an address into the collection. The merged set would refuse it + anyway, but without saying which sets collided. + """ + if "id" not in frame.columns: + return + # Blank spelled as the set spells it, so this cannot drift from what + # `_check_ids` counts as an unstated id. + ids = frame["id"].map(_text) + shared = (ids != "") & ids.duplicated(keep=False) + if not shared.any(): + return + first = ids[shared].iloc[0] + named = sorted(set(frame.loc[ids == first, "set"])) + msg = ( + f"The annotation id {first} names a row in {' and '.join(named)}. Ids are " + "unique across the sets loaded together, so one id names one annotation." + ) + raise ParameterError(msg) + + +def _load_set(directory: Path, attrs: Mapping, dims, **kwargs) -> AnnotationSet: + """Load the set a directory holds.""" + # Dimensions are the directory's once it has stated them: reading the # cells against other dimensions would type them differently and # build a set which is not the one stored here. if attrs.get("dims"): _refuse_overrides("a directory stating its own dimensions", dims=dims) - _refuse_stray_tables(directory) - table = directory / f"{ANNOTATION_STEM}{TABLE_SUFFIX}" - if not table.exists(): + _refuse_stray_tables( + directory, + (ANNOTATION_STEM, VERTEX_STEM), + f"which name no part of a set. A set states {ANNOTATION_STEM} and, " + f"where it has vertices, {VERTEX_STEM}, each as a " + f"{' or a '.join(TABLE_SUFFIXES)} table.", + ) + table = _one_spelling(directory, ANNOTATION_STEM, TABLE_SUFFIXES) + if table is None: msg = ( - f"{quote_path(directory)} holds no {ANNOTATION_STEM}{TABLE_SUFFIX}, " - "so it states no annotations." + f"{quote_path(directory)} holds no {ANNOTATION_STEM} table and no " + f"{BLESSED_NAME}, so it states no annotations and carries none." ) raise ParameterError(msg) - stated = _declared_dims(attrs, dims, directory) - frame = _read_set_table(table, stated, "no annotations") - vertex_path = directory / f"{VERTEX_STEM}{TABLE_SUFFIX}" + declared, skip = _read_table_dims(table) + stated = _declared_dims(attrs, dims, directory, declared, table) + frame = _read_set_table(table, stated, "no annotations", skip=skip) + # Found as the annotations table is: a set spells each of its parts + # once, and a `vertices.CSV` beside a `vertices.csv` is two spellings of + # one part rather than a table nobody reads. + vertex_path = _one_spelling(directory, VERTEX_STEM, TABLE_SUFFIXES) vertices = None - if vertex_path.exists(): - vertices = _read_set_table(vertex_path, stated, "no vertices", ordered=True) + if vertex_path is not None: + vertices = _read_set_table( + vertex_path, + stated, + "no vertices", + ordered=True, + skip=_vertex_declaration(vertex_path), + ) # The read dimensions rather than the given ones: they are the same # names, already a tuple, so a file spelling one dimension as a bare # string builds the set the same way the constructor would. @@ -333,34 +1024,90 @@ def _load_directory(directory: Path, dims, **kwargs) -> AnnotationSet: def _load_file(path: Path, dims, **kwargs) -> AnnotationSet: """Load the set a bare table holds.""" - if path.suffix.lower() != TABLE_SUFFIX: + if path.suffix.casefold() not in TABLE_SUFFIXES: + named = " or ".join(TABLE_SUFFIXES) msg = ( f"{quote_path(path)} is not a table an annotation set is read from. " - f"A bare set is a {TABLE_SUFFIX} file; a set with vertices is a " - "directory." + f"A bare set is a {named} file; a set with vertices is a directory." ) raise ParameterError(msg) - stated = _declared_dims({}, dims, path) + declared, skip = _read_table_dims(path) + # A bare table states no attributes of its own, so a caller may hand it + # some -- and the dimensions they name are the ones its cells are read + # in, since nothing can be read before that is known. + stated = _declared_dims(_given_attrs(kwargs), dims, path, declared, path) return AnnotationSet( - _read_set_table(path, stated, "no annotations"), dims=stated, **kwargs + _read_set_table(path, stated, "no annotations", skip=skip), + dims=stated, + **kwargs, ) -def _declared_dims(attrs: Mapping, dims, source: Path) -> tuple[str, ...]: +def _vertex_declaration(path: Path) -> int: + """ + Refuse a vertices table which declares dimensions, and return the lines + above its header -- which, since it declares none, is none of them. + + Vertices are read in the dimensions of the set they belong to, whether + they would declare them above a header or in a footer. The lines above a + header are skipped only where a declaration is among them, so a vertices + table has no preamble to skip. + """ + declared, skip = _read_table_dims(path) + if declared is not None: + where = ( + f"states {DIMS_KEY}" + if _is_parquet(path) + else f"declares {_DIMS_PRAGMA} above its header" + ) + msg = ( + f"{quote_path(path)} {where}. Vertices are read in the dimensions " + "of the set they belong to, which states them once." + ) + raise ParameterError(msg) + return skip + + +def _declared_dims( + attrs: Mapping, + dims, + source: Path, + declared: tuple[str, ...] | None = None, + table: Path | None = None, +) -> tuple[str, ...]: """ Return the dimensions to read a source in: the caller's, else its own. The cells cannot be read before this is known -- which columns hold times rather than text is exactly what a dimension decides -- so a source stating none fails here rather than as a puzzling column later. + + Where a table declares them above its header, restating them is allowed + if the two agree and refused if they differ: there is no precedence rule + between two spellings of one fact. """ stated = dims if dims is not None else attrs.get("dims") + if declared is not None: + if stated is None: + stated = declared + elif tuple(str(x) for x in iterate(stated)) != declared: + spelled = "was given" if dims is not None else "is stated in its attributes" + named = table or source + where = f"in its {DIMS_KEY}" if _is_parquet(named) else "above its header" + msg = ( + f"{quote_path(named)} declares the dimensions " + f"{', '.join(declared)} {where}, but " + f"{', '.join(str(x) for x in iterate(stated))} {spelled}. " + "Dimensions may be spelled twice where the two agree." + ) + raise ParameterError(msg) if not stated: msg = ( f"{quote_path(source)} states no dimensions, and none were given. " "Annotations are read in the dimensions they are stated in: write " - f"them in {ATTRS_STEM}{OBJECT_SUFFIXES[0]} or pass " - "dims=('distance', 'time')." + f"them in {ATTRS_STEM}{OBJECT_SUFFIXES[0]}, in a " + f"'{_COMMENT} {_DIMS_PRAGMA}: distance, time' line above the table, " + "or pass dims=('distance', 'time')." ) raise ParameterError(msg) # Through `iterate`, as the set itself reads them: a lone string is one @@ -387,11 +1134,14 @@ def annotations( Parameters ---------- source - An `AnnotationSet`, a path to a set directory or a CSV table, or a - dataframe of one row per annotation. + An `AnnotationSet`, a dataframe of one row per annotation, or a path + to a CSV table, a set directory, a directory of set directories, or a + directory of data carrying either under ``.annotations``. dims The patch dimensions the annotations are stated in. Required unless - the source states them itself, and overriding it where it does. + the source states them itself -- in its attributes, or in a + ``# dims: distance, time`` line above its table. A source which + states them takes them again only if the two agree. **kwargs Passed to [`AnnotationSet`](`dascore.core.annotations.AnnotationSet`). A source already holding what one states -- a set, or a directory @@ -412,6 +1162,29 @@ def annotations( >>> dc.annotations(picks) is picks True + + Sets stored side by side read as one, each row saying which set it came + from. + + >>> import tempfile + >>> from pathlib import Path + >>> with tempfile.TemporaryDirectory() as folder: + ... root = Path(folder) / "sets" + ... _ = picks.io.save(root / "hand") + ... _ = picks.io.save(root / "phasenet") + ... together = dc.annotations(root) + >>> len(together), together[0].set, sorted(together.attrs.sets) + (2, 'hand', ['hand', 'phasenet']) + + A directory of data is read as the annotations it carries, so the + directory a spool was opened on is a path this takes too. + + >>> with tempfile.TemporaryDirectory() as folder: + ... data = Path(folder) + ... _ = picks.io.save(data / ".annotations") + ... carried = dc.annotations(data) + >>> carried == picks + True """ if isinstance(source, AnnotationSet): # A built set states everything these would override, and building @@ -422,10 +1195,8 @@ def annotations( if isinstance(source, str | os.PathLike): path = Path(source) try: - if path.is_dir(): - return _load_directory(path, dims, **kwargs) if path.exists(): - return _load_file(path, dims, **kwargs) + return _load_path(path, dims, **kwargs) # ValidationError too: a stored document which does not build the # models is a bad file, and it is named as one here rather than # arriving as pydantic's report on a call the caller did not make. diff --git a/dascore/core/annotations.py b/dascore/core/annotations.py index b7265d7b6..657904373 100644 --- a/dascore/core/annotations.py +++ b/dascore/core/annotations.py @@ -55,7 +55,12 @@ from dascore.utils.mapping import FrozenDict from dascore.utils.misc import iterate, to_str, validate_acquisition_key from dascore.utils.namespace import NamespaceOwner -from dascore.utils.tables import parse_cell +from dascore.utils.tables import ( + parquet_table, + parse_cell, + write_parquet, + write_parquet_table, +) from dascore.utils.time import to_datetime64, to_timedelta64 # Columns any set may carry, whatever dimensions it declares. @@ -68,6 +73,7 @@ "geometry", "basis", "acquisition_key", + "set", ) # The geometries a row may declare. A region is the default: it is what @@ -87,9 +93,20 @@ ATTRS_STEM = "attrs" ANNOTATION_STEM = "annotations" VERTEX_STEM = "vertices" -TABLE_SUFFIX = ".csv" +# The encodings a table takes, the suffix naming which one a file holds. +# CSV is the floor: it needs nothing beyond the standard library, so a set +# can always be written. Parquet is the same tables with their types kept, +# for a set too big to want text; it needs pyarrow. +TABLE_SUFFIXES = (".csv", ".parquet") +TABLE_SUFFIX = TABLE_SUFFIXES[0] OBJECT_SUFFIXES = (".json", ".yaml", ".yml") +# What a parquet table names its dimensions in, since it has no comment +# line to declare them in and its footer is the place a format states what +# its columns cannot. A JSON document, as GeoParquet's `geo` key holds one; +# namespaced, so a file may carry both without either reading the other's. +DIMS_KEY = "dascore:dims" + # What a range column is spelled with. _START, _END = "_start", "_end" @@ -508,6 +525,14 @@ class Annotation(_AnnotationModel): "set's own where the row names none." ), ) + set: str = Field( + default="", + description=( + "Name of the set this annotation was read from, where many were " + "loaded together. A label, not an identity: ids are unique across " + "a collection." + ), + ) extra: FrozenDictType[str, Any] = Field( default_factory=dict, description="Columns the set does not model." ) @@ -571,6 +596,19 @@ class AnnotationSetAttrs(_AnnotationModel): columns: FrozenDictType[str, AnnotationColumn] = Field( default_factory=dict, description="Documentation for columns, keyed by name." ) + sets: FrozenDictType[str, AnnotationSetAttrs] = Field( + default_factory=dict, + description=( + "The attributes of each set loaded together, keyed by the name the " + "`set` column holds. What a child set declares for itself -- its " + "own dimensions, provenance and columns -- is kept here rather " + "than written into every row of it, and a row reaches it back " + "through its label. What it says describes its own table, not the " + "merged one: a column of whole numbers which another set does not " + "state holds them as floats once the two are one table, since that " + "is what a missing number makes of them." + ), + ) @model_validator(mode="after") def _check_dims(self) -> Self: @@ -598,6 +636,25 @@ def _check_dims(self) -> Self: raise ValueError(msg) return self + @model_validator(mode="after") + def _check_sets(self) -> Self: + """Sets loaded together are one collection, in its dimensions.""" + for name, child in self.sets.items(): + if child.sets: + msg = ( + f"The set {name!r} states sets of its own. Sets loaded " + "together are one collection, not a tree of them." + ) + raise ValueError(msg) + if extra := sorted(set(child.dims) - set(self.dims)): + msg = ( + f"The set {name!r} states the dimension(s) " + f"{', '.join(extra)}, which the sets loaded with it do not: " + f"they hold {list(self.dims)}." + ) + raise ValueError(msg) + return self + # --- The set -------------------------------------------------------------- @@ -675,6 +732,7 @@ def __init__( _check_columns(frame, self._attrs) _check_ranges(frame, spellings) _check_values(frame) + _check_set_labels(frame, self._attrs) ids = _check_ids(frame) frame = _normalize_tags(_normalize_basis(frame, self._attrs.dims)) vertex_frame = _normalize_times( @@ -712,6 +770,7 @@ def __iter__(self): def __getitem__(self, position: int) -> Annotation: """Return one annotation by its position.""" row = self._df.iloc[position] + label = _text(row.get("set")) return Annotation( geometry=self._geometry(row), id=_text(row.get("id")), @@ -719,13 +778,30 @@ def __getitem__(self, position: int) -> Annotation: value=row["value"] if _stated(row.get("value")) else True, tags=_read_tags(row.get("tags")), parent=_text(row.get("parent")), - # A set may span acquisitions, so a row naming one overrides - # the set-level address rather than sitting beside it. - acquisition_key=_text(row.get("acquisition_key")) - or self._attrs.acquisition_key, + acquisition_key=self._acquisition_key(row, label), + set=label, extra=_read_extra(row, self._attrs.dims, self._spellings), ) + def _acquisition_key(self, row, label: str) -> str: + """ + Return the address of the data one annotation was made on. + + A set may span acquisitions, so a row naming one overrides the + set-level address rather than sitting beside it. Where sets were + loaded together, the row's own set answers before the collection + does: the collection is not what any of them was picked on, and the + label is what reaches back to the set which was. A collection which + states an address of its own still answers for a set which states + none, which is what makes stating it once useful. + """ + if stated := _text(row.get("acquisition_key")): + return stated + child = self._attrs.sets.get(label) + if child is not None and child.acquisition_key: + return child.acquisition_key + return self._attrs.acquisition_key + def __eq__(self, other) -> bool: """Two sets are equal when their attributes and frames are.""" if not isinstance(other, AnnotationSet): @@ -932,6 +1008,43 @@ def _check_ranges(frame: pd.DataFrame, spellings) -> None: raise ParameterError(msg) +def _check_set_labels(frame: pd.DataFrame, attrs: AnnotationSetAttrs) -> None: + """ + Refuse a row whose set label names none of the sets stated. + + Sets loaded together keep their rows in one table and what each of them + states in ``attrs.sets``; a row whose label names no set -- or names + nothing at all -- has lost that half, and would quietly answer with the + collection's provenance rather than its own. Only checked where sets are + stated: a set on its own may carry a `set` column meaning whatever it + means, and a collection which happens to hold no rows labels none. + """ + if not attrs.sets or frame.empty: + return + stated = ", ".join(sorted(attrs.sets)) + if "set" not in frame.columns: + msg = ( + f"This states the sets {stated} and no set column, so no row says " + "which of them it came from." + ) + raise ParameterError(msg) + labels = frame["set"].map(_text) + if not labels.all(): + rows = ", ".join(str(x) for x in frame.index[labels == ""][:5]) + msg = ( + f"Row(s) {rows} state no set, where the sets {stated} are stated. A " + "row loaded with others says which of them it came from." + ) + raise ParameterError(msg) + if unknown := sorted(set(labels) - set(attrs.sets)): + msg = ( + f"The set label(s) {', '.join(unknown)} name no set stated here, " + f"which states {stated}. A label reaches back to what its set says " + "about itself, so it names one of them." + ) + raise ParameterError(msg) + + def _check_values(frame: pd.DataFrame) -> None: """ Refuse a group whose values are not all one kind. @@ -1385,6 +1498,40 @@ def _refuse_ambiguous_values(frame: pd.DataFrame) -> None: raise ParameterError(msg) +def _table_suffix(format: str) -> str: + """Return the suffix an encoding is named by, refusing an unknown one.""" + suffix = f".{str(format).lower().lstrip('.')}" + if suffix not in TABLE_SUFFIXES: + named = ", ".join(x.lstrip(".") for x in TABLE_SUFFIXES) + msg = f"{format!r} is not a table encoding; a set is written as {named}." + raise ParameterError(msg) + return suffix + + +def _spell_table(frame: pd.DataFrame, suffix: str, dims: Sequence[str] | None = None): + """ + Spell one table for the encoding its suffix names. + + Spelled before the directory is touched, so whichever encoding is + asked for, a table which cannot be written raises with the stored set + still whole. + """ + if suffix == TABLE_SUFFIX: + return _write_table(frame) + # A parquet file has no comment line to declare its dimensions in, so + # they go in the metadata its footer holds. + metadata = None if dims is None else {DIMS_KEY: json.dumps(list(dims))} + return parquet_table(frame, metadata) + + +def _write_spelled(payload, path) -> None: + """Write what `_spell_table` spelled, whichever encoding it is.""" + if isinstance(payload, str): + _write_text(payload, path) + else: + write_parquet_table(payload, path) + + def _write_table(frame: pd.DataFrame, path=None) -> str: """Return a frame as CSV text, optionally writing it to a path.""" _refuse_ambiguous_values(frame) @@ -1546,8 +1693,13 @@ def annotation_set_to_csv( A bare table states one grain, so a set holding vertices is written with [save](`dascore.core.annotations.save_annotation_set`) instead; - this is the spelling for a set of regions. The set's dimensions are - not part of the table, so reading one back states them again. + this is the spelling for a set of regions. + + The dimensions are not written: they are not a column, and the only + place a CSV has for them is a comment line, which a reader not told + to expect one takes for the header. Reading such a table back states + them again, in the call or in a ``# dims: distance, time`` line + written above the header by hand. Parameters ---------- @@ -1567,6 +1719,62 @@ def annotation_set_to_csv( >>> "group" in annotations.io.to_csv() True """ + _refuse_bare_vertices(annotations) + return _write_table(annotations._df, path) + + +def annotation_set_to_parquet( + annotations: AnnotationSet, path: str | pathlib.Path +) -> pathlib.Path: + """ + Write the annotations as one parquet file. + + Reached as ``annotation_set.io.to_parquet``. + + The parquet spelling of + [to_csv](`dascore.core.annotations.annotation_set_to_csv`), for a + set too big to want text. It keeps what a CSV cannot: a column + parquet has a type for comes back as that type rather than as a + spelling to be guessed at, and the dimensions travel in the file's + own metadata rather than having to be stated again. A column with + no one type is written as JSON, which keeps the value of each cell + but not every python type it may have been held in -- a tuple comes + back as a list. + + Needs pyarrow, which CSV does not; a set of regions can always be + written as a table, whatever is installed. + + Parameters + ---------- + annotations + The set to write. + path + Where to write the file. + + Returns + ------- + The path written to, so a save reads straight back. + + Examples + -------- + >>> import pandas as pd + >>> import dascore as dc + >>> frame = pd.DataFrame( + ... {"group": ["event"], "distance_start": [10.0], "distance_end": [80.0]} + ... ) + >>> annotations = dc.AnnotationSet(frame, dims=("time", "distance")) + >>> path = annotations.io.to_parquet("picks.parquet") # doctest: +SKIP + >>> dc.annotations(path) == annotations # doctest: +SKIP + True + """ + _refuse_bare_vertices(annotations) + dims = json.dumps(list(annotations.dims)) + write_parquet(annotations._df, path, {DIMS_KEY: dims}) + return pathlib.Path(path) + + +def _refuse_bare_vertices(annotations: AnnotationSet) -> None: + """Refuse to write a set of shapes as one table, which has one grain.""" if not annotations._vertices.empty: msg = ( "This set holds vertices, which a bare table has no row for. " @@ -1574,11 +1782,10 @@ def annotation_set_to_csv( "annotations." ) raise ParameterError(msg) - return _write_table(annotations._df, path) def save_annotation_set( - annotations: AnnotationSet, path: str | pathlib.Path + annotations: AnnotationSet, path: str | pathlib.Path, format: str = "csv" ) -> pathlib.Path: """ Write the set to a directory, creating it if needed. @@ -1594,9 +1801,20 @@ def save_annotation_set( a set needs nothing beyond the standard library; a set authored by hand may spell them in YAML, which reads back the same. + Sets which were loaded together write one table rather than a + directory each: the ``set`` column already says which set every row + belongs to, and what each of them states for itself travels in the + attributes, so the flat spelling loses nothing. + + The tables are CSV unless another encoding is asked for. Parquet + writes the same parts under the same names, with its own suffix, and + keeps a column's type rather than its spelling wherever it has one + for it; it needs pyarrow, where CSV needs nothing. + Writing states the whole directory, so a part this set does not - have is removed rather than left behind. A stale vertices table, or - the YAML the attributes used to be spelled in, would otherwise sit + have is removed rather than left behind. A stale vertices table, the + YAML the attributes used to be spelled in, or the CSV a set was + written as before it was written as parquet, would otherwise sit beside what was written and leave a directory which loaded before the save refusing to load after it. @@ -1606,6 +1824,8 @@ def save_annotation_set( The set to write. path The directory to write into. + format + The encoding the tables are written in: ``csv`` or ``parquet``. Returns ------- @@ -1630,33 +1850,54 @@ def save_annotation_set( # Defaults are dropped from the document, so it says what the set # says; dims has no default, so it is always written, and the # attributes name their own model, which is what the file holds. + suffix = _table_suffix(format) document = annotations._attrs.model_dump(mode="json", exclude_defaults=True) - annotation_text = _write_table(annotations._df) - vertex_text = ( - None if annotations._vertices.empty else _write_table(annotations._vertices) - ) + dims = annotations.dims + spelled = {ANNOTATION_STEM: _spell_table(annotations._df, suffix, dims)} + if not annotations._vertices.empty: + spelled[VERTEX_STEM] = _spell_table(annotations._vertices, suffix) directory = pathlib.Path(path) directory.mkdir(parents=True, exist_ok=True) - vertex_table = directory / f"{VERTEX_STEM}{TABLE_SUFFIX}" attrs_file = directory / f"{ATTRS_STEM}{OBJECT_SUFFIXES[0]}" + writing = {attrs_file, *(directory / f"{x}{suffix}" for x in spelled)} # Only the spellings this format claims: a notes.txt or an # attrs.bak beside them participates in no convention and is not # this function's to delete. The suffix is matched without regard # to case, as the loader matches it. + claimed = { + ATTRS_STEM: OBJECT_SUFFIXES, + ANNOTATION_STEM: TABLE_SUFFIXES, + VERTEX_STEM: TABLE_SUFFIXES, + } superseded = [ x for x in directory.iterdir() - if x.stem == ATTRS_STEM - and x.suffix.casefold() in OBJECT_SUFFIXES - and x != attrs_file + if x.suffix.casefold() in claimed.get(x.stem, ()) and x not in writing ] - if vertex_text is None: - superseded.append(vertex_table) - for stale in superseded: - stale.unlink(missing_ok=True) + # Written before the superseded parts are cleared, not after: a + # write which fails partway -- a full disk, a permission changed + # under it -- then leaves the set it was replacing still in the + # directory, and a reader finds two spellings of one part and says + # so, rather than finding the set gone. with open(attrs_file, "w") as stream: json.dump(document, stream, indent=2) - _write_text(annotation_text, directory / f"{ANNOTATION_STEM}{TABLE_SUFFIX}") - if vertex_text is not None: - _write_text(vertex_text, vertex_table) + for stem, payload in spelled.items(): + _write_spelled(payload, directory / f"{stem}{suffix}") + for stale in superseded: + # A part just written is not stale under another name: a + # case-insensitive filesystem holds `attrs.JSON` and the + # `attrs.json` written over it in one file, and unlinking the + # older spelling there would take the set with it. + if any(_one_file(stale, x) for x in writing): + continue + stale.unlink(missing_ok=True) return directory + + +def _one_file(one: pathlib.Path, other: pathlib.Path) -> bool: + """Whether two names reach one file, as a case-insensitive store lets them.""" + try: + return one.samefile(other) + except OSError: + # One of them is gone, so they are not the same file. + return False diff --git a/dascore/io/__init__.py b/dascore/io/__init__.py index fbc75fde3..72916be20 100644 --- a/dascore/io/__init__.py +++ b/dascore/io/__init__.py @@ -21,6 +21,7 @@ from dascore.core.annotations import ( annotation_set_to_csv, annotation_set_to_dataframe, + annotation_set_to_parquet, annotation_set_to_vertices, save_annotation_set, ) @@ -60,4 +61,5 @@ class AnnotationIO(AnnotationNameSpace): to_dataframe = annotation_set_to_dataframe to_vertices = annotation_set_to_vertices to_csv = annotation_set_to_csv + to_parquet = annotation_set_to_parquet save = save_annotation_set diff --git a/dascore/utils/pd.py b/dascore/utils/pd.py index be9dd106f..19c5874c4 100644 --- a/dascore/utils/pd.py +++ b/dascore/utils/pd.py @@ -393,7 +393,13 @@ def _filter_contains(query_dict, df, bool_index): """Filter based on rows containing specified values.""" for key, val in query_dict.items(): _check_misdirected_range_query(key, val, df) - bool_index = np.logical_and(bool_index, df[key].isin(val)) + # An ellipsis names no value, so it matches nothing and is dropped + # before `isin` sees it: pandas' arrow-backed string columns refuse a + # value arrow has no type for, where its numpy-backed ones quietly + # never match. The range check above still sees it -- an open bound + # in a membership query is what that check exists to name. + wanted = [x for x in val if x is not ...] + bool_index = np.logical_and(bool_index, df[key].isin(wanted)) return bool_index diff --git a/dascore/utils/tables.py b/dascore/utils/tables.py index 2f4c8a2dc..7e2769ce1 100644 --- a/dascore/utils/tables.py +++ b/dascore/utils/tables.py @@ -14,15 +14,24 @@ from __future__ import annotations import csv +import datetime +import json +from collections.abc import Mapping, Sized from pathlib import Path +import numpy as np import pandas as pd from dascore.exceptions import ParameterError +from dascore.utils.misc import optional_import, to_str from dascore.utils.paths import quote_path +from dascore.utils.time import to_datetime64, to_timedelta64 +# The metadata key a parquet file names its document columns in. +DOCUMENT_KEY = "dascore:documents" -def read_table(path: Path, what: str = "nothing") -> pd.DataFrame: + +def read_table(path: Path, what: str = "nothing", skip: int = 0) -> pd.DataFrame: r""" Read one strict CSV table. @@ -37,6 +46,12 @@ def read_table(path: Path, what: str = "nothing") -> pd.DataFrame: The CSV file to read. what What a table with no columns fails to state, for that error message. + skip + Lines above the header, for a format which states something of its + own before its table. Read past before the table is parsed at all, + so nothing downstream has to agree about what those lines were. Row + numbers in errors still count from the top of the file, so they name + the line a reader would look at. Examples -------- @@ -55,8 +70,17 @@ def read_table(path: Path, what: str = "nothing") -> pd.DataFrame: # frame exists the second one is `coupling_type.1` and the clash cannot # be seen; and it raises its own error for a file with no columns, # which would arrive before this one could say what was expected. + # + # One open, which pandas then reads on from: `skiprows` would have it + # skip *rows*, and a quote in a skipped line can make one row span the + # rest of the file, leaving pandas a different header than was checked + # here. Both readers therefore also decode alike, which a locale-encoded + # read or a byte order mark reaching only one of them would break. try: with path.open(newline="", encoding="utf-8-sig") as stream: + for _ in range(skip): + stream.readline() + position = stream.tell() reader = csv.reader(stream) header = next(reader, []) if header: @@ -64,13 +88,28 @@ def read_table(path: Path, what: str = "nothing") -> pd.DataFrame: # format meant to grow, and holding every cell as a python # object beside the frame pandas builds would cost several # times what the frame itself does. - _check_widths(reader, header, path) + _check_widths(reader, header, path, start=skip + 2) + _check_header(header, path, what) + stream.seek(position) + # index_col=False so that no column is ever read as an index; the + # row widths above already agree, and this keeps them agreeing. + return pd.read_csv( + stream, + dtype=str, + keep_default_na=False, + na_values=[""], + index_col=False, + ) # csv.Error too: a cell longer than csv.field_size_limit stops the # header scan, and without this it would leave this function as a bare # _csv.Error rather than as whatever the caller's format raises. except (OSError, UnicodeDecodeError, csv.Error) as read_error: msg = f"Could not read {quote_path(path)}: {read_error}." raise ParameterError(msg) from read_error + + +def _check_header(header: list[str], path: Path, what: str) -> None: + """Refuse a table with no columns, or one which names a column twice.""" if not header: msg = f"{quote_path(path)} has no columns, so it states {what}." raise ParameterError(msg) @@ -81,22 +120,9 @@ def read_table(path: Path, what: str = "nothing") -> pd.DataFrame: "column states one field." ) raise ParameterError(msg) - # index_col=False so that no column is ever read as an index; the row - # widths above already agree, and this keeps them agreeing. - return pd.read_csv( - path, - dtype=str, - keep_default_na=False, - na_values=[""], - index_col=False, - # Both readers decode alike, or the header checked above is not - # the header parsed here: a locale-encoded read disagrees with - # pandas' UTF-8, and a byte order mark reaches only one of them. - encoding="utf-8-sig", - ) -def _check_widths(reader, header: list[str], path: Path) -> None: +def _check_widths(reader, header: list[str], path: Path, start: int = 2) -> None: """ Refuse a row which is not its header wide. @@ -105,7 +131,7 @@ def _check_widths(reader, header: list[str], path: Path) -> None: the row shifts one field left and lands in its neighbour's meaning. A row states one cell per column or it is not a row. """ - for number, row in enumerate(reader, start=2): + for number, row in enumerate(reader, start=start): if row and len(row) != len(header): msg = ( f"{quote_path(path)} row {number} states {len(row)} cells where " @@ -114,6 +140,320 @@ def _check_widths(reader, header: list[str], path: Path) -> None: raise ParameterError(msg) +def write_parquet(frame: pd.DataFrame, path, metadata: Mapping | None = None) -> None: + """ + Write a dataframe as one parquet file, keeping the values it holds. + + Parquet stores types, so a column comes back as what it was written as + rather than as text a reader has to guess at. A column with no single + type -- one holding both text and booleans, or a model, or a nested + mapping -- has no parquet type of its own; each of its cells is written + as a JSON document instead, and the file names those columns in its + metadata so a reader gets the values back rather than their spelling. + + Parameters + ---------- + frame + The table to write. + path + Where to write it. + metadata + Key-value strings for the file's own metadata, which + [read_parquet](`dascore.utils.tables.read_parquet`) hands back. A + format states here what its table cannot say in a column. + + Examples + -------- + Needs pyarrow, which the doctest run does not have, so this one is not + executed; `tests/test_utils/test_tables.py` runs the same round trip. + + >>> import tempfile # doctest: +SKIP + >>> from pathlib import Path + >>> import pandas as pd + >>> from dascore.utils.tables import read_parquet, write_parquet + >>> frame = pd.DataFrame({"group": ["rail"], "value": [True]}) + >>> with tempfile.TemporaryDirectory() as folder: # doctest: +SKIP + ... path = Path(folder) / "coupling.parquet" + ... write_parquet(frame, path, {"dascore:dims": "distance"}) + ... out, stated = read_parquet(path) + >>> out.equals(frame), stated["dascore:dims"] # doctest: +SKIP + (True, 'distance') + """ + write_parquet_table(parquet_table(frame, metadata), path) + + +def parquet_table(frame: pd.DataFrame, metadata: Mapping | None = None): + """ + Return a dataframe as the parquet table it is written from. + + Spelled out before anything is written, so a caller storing several + tables at once can prepare them all before it touches the directory. + See [write_parquet](`dascore.utils.tables.write_parquet`), which is + this and the write together. + """ + arrow = optional_import("pyarrow", required_for="parquet tables") + spelled, documents = {}, [] + for label, name in zip(frame.columns, _named(frame.columns), strict=True): + series = frame[label] + if _one_type(series): + spelled[name] = series + continue + documents.append(name) + spelled[name] = series.map(_document) + stated = {str(k): str(v) for k, v in (metadata or {}).items()} + if reserved := sorted(x for x in stated if _is_reserved(x)): + msg = ( + f"The metadata key(s) {', '.join(reserved)} are not a caller's to " + f"state: {DOCUMENT_KEY} is what this writer names its own document " + "columns in, and pandas and ARROW: are what the file already holds " + "for the reader which wrote them." + ) + raise ParameterError(msg) + if documents: + stated[DOCUMENT_KEY] = json.dumps(documents) + table = arrow.Table.from_pandas( + pd.DataFrame(spelled, index=frame.index), preserve_index=False + ) + # Added to what pyarrow wrote rather than replacing it: the pandas key + # it puts there is what a pandas reader uses to rebuild an index and the + # dtypes it can, and dropping it would make this file say less than + # pyarrow wrote. + kept = {**(table.schema.metadata or {}), **stated} + return table.replace_schema_metadata(kept) + + +def write_parquet_table(table, path) -> None: + """Write a prepared parquet table, as `parquet_table` returns it.""" + parquet = optional_import("pyarrow.parquet", required_for="parquet tables") + parquet.write_table(table, path) + + +def read_parquet( + path, what: str = "nothing", empty: bool = False +) -> tuple[pd.DataFrame, dict[str, str]]: + """ + Read one parquet file, and whatever it states about itself. + + Returns the table and its metadata, with the columns + [write_parquet](`dascore.utils.tables.write_parquet`) stored as JSON + documents read back as the values they hold. + + Parameters + ---------- + path + The parquet file to read. + what + What a table with no columns fails to state, for that error message. + empty + Whether a table with no columns is allowed, for a format in which + that is how something holding nothing is written. + """ + parquet = optional_import("pyarrow.parquet", required_for="parquet tables") + try: + table = parquet.read_table(path) + stated = _stated_metadata(table.schema.metadata) + # Inside the block as well: pandas metadata another writer left is + # read here, and a malformed one raises where nothing else would + # name the file it came from. + frame = table.to_pandas() + except Exception as error: + # Any error pyarrow raises: it reports a truncated file, an + # unreadable one and something which is not parquet at all through + # several types of its own, and the caller's format names them all + # the same way. + msg = f"Could not read {quote_path(path)}: {error}." + raise ParameterError(msg) from error + if not len(frame.columns) and not empty: + msg = f"{quote_path(path)} has no columns, so it states {what}." + raise ParameterError(msg) + for name in _document_columns(stated.pop(DOCUMENT_KEY, "[]"), path): + if name not in frame.columns: + msg = ( + f"{quote_path(path)} names {name!r} as a column of documents, " + "and holds no such column." + ) + raise ParameterError(msg) + # Held as object: the cells are whatever their documents state, and + # letting pandas re-infer a type from them would hand back a column + # of a type the file never said it had. + read = [_read_document(x, name, path) for x in frame[name]] + frame[name] = pd.Series(read, index=frame.index, dtype=object) + return frame, stated + + +def read_parquet_metadata(path) -> dict[str, str]: + """ + Return what a parquet file states about itself, reading no rows. + + The footer alone, so a caller which needs what a file says before it + decides how to read the table -- which dimensions its columns are in, + say -- does not pay for the table to find out. + + Parameters + ---------- + path + The parquet file to read. + """ + parquet = optional_import("pyarrow.parquet", required_for="parquet tables") + try: + schema = parquet.read_schema(path) + except Exception as error: + msg = f"Could not read {quote_path(path)}: {error}." + raise ParameterError(msg) from error + return _stated_metadata(schema.metadata) + + +def _document_columns(stated: str, path) -> list[str]: + """Read the columns a file names as documents, refusing what it cannot mean.""" + try: + names = json.loads(stated) + except ValueError as error: + msg = ( + f"{quote_path(path)} states {DOCUMENT_KEY} as {stated!r}, which is " + f"not a JSON document: {error}." + ) + raise ParameterError(msg) from error + if not isinstance(names, list) or not all(isinstance(x, str) for x in names): + msg = ( + f"{quote_path(path)} states {DOCUMENT_KEY} as {stated!r}; it names " + "the columns which hold documents, so it is a list of names." + ) + raise ParameterError(msg) + return names + + +def _named(columns) -> list[str]: + """ + Return a table's column names as parquet holds them, which is as text. + + A frame may label a column with anything hashable and a CSV writes + whatever that prints as, but arrow takes names only as strings -- so + they are spelled here rather than left to fail deep inside a conversion. + Two labels which spell alike would name one column, which is the same + refusal a frame naming a column twice already gets. + """ + named = [str(x) for x in columns] + if len(set(named)) != len(named): + repeated = sorted({x for x in named if named.count(x) > 1}) + msg = ( + f"The column(s) {', '.join(repeated)} are named more than once " + "when their names are spelled as text, which is how parquet holds " + "them; one column states one thing." + ) + raise ParameterError(msg) + return named + + +def _is_reserved(key: str) -> bool: + """Whether a metadata key names something the file states for itself.""" + return key == DOCUMENT_KEY or key == "pandas" or key.startswith("ARROW:") + + +def _one_type(series: pd.Series) -> bool: + """Whether a column holds one type parquet has a column shape for.""" + if series.dtype != object: + return True + # Text is the one type an object column may still hold: pandas gives + # some string columns object dtype, and a whole column of text is + # exactly what parquet stores as a string column. + return all(isinstance(x, str) for x in series if _is_stated(x)) + + +def _is_stated(value) -> bool: + """Whether a cell states anything, whatever kind of thing it holds.""" + if value is None: + return False + if isinstance(value, Sized) and not isinstance(value, str): + # A container states itself; asking pandas whether it is null + # answers once per element, which has no truth value. + return True + return not pd.isna(value) + + +def _document(value): + """Spell one cell of a column parquet has no single type for.""" + if not _is_stated(value): + return None + return json.dumps(_documented(value), default=str) + + +def _documented(value): + """ + Return a value as the json types it is made of. + + A missing value nested inside one becomes null rather than the text of + whatever spelling of missing it was: `pd.NA` written as "" would + come back as a string a reader takes for a value. + + Numpy's scalars are the ones that matter: `np.int64` is not an `int` + and `np.bool_` is not a `bool`, so json falls back to spelling them as + text -- and a column of numbers written by numpy would come back as a + column of strings, which is exactly what this encoding exists not to + do. A time has no json type at all and is spelled as DASCore spells + every time, which reads back as one. + + A value none of this can spell -- a `Decimal`, a `bytes`, an object of + someone's own class -- is written as its text, which is what the CSV + encoding does with it too. That is the one place this loses a type, and + the reason a column parquet has a type for is never sent this way. + """ + if not _is_stated(value): + return None + # At nanoseconds, whatever resolution the value arrived in: a set holds + # its times at nanoseconds, and one spelling per instant is what makes + # two writes of one set the same file. + if isinstance(value, np.datetime64): + return to_str(np.datetime64(value, "ns")) + if isinstance(value, np.timedelta64): + return to_str(np.timedelta64(value, "ns")) + if hasattr(value, "model_dump"): + return value.model_dump(mode="json") + if isinstance(value, np.generic): + return _documented(value.item()) + if isinstance(value, datetime.datetime | datetime.date | pd.Timestamp): + return _documented(to_datetime64(value)) + # `pd.Timedelta` is one of these, as `pd.Timestamp` is a datetime. + if isinstance(value, datetime.timedelta): + return _documented(to_timedelta64(value)) + if isinstance(value, Mapping): + return {str(k): _documented(v) for k, v in value.items()} + if isinstance(value, list | tuple | set | frozenset | np.ndarray): + return [_documented(x) for x in value] + return value + + +def _read_document(cell, name: str, path) -> object: + """Read one cell of a column the file names as documents.""" + if not isinstance(cell, str): + # Every spelling of a null becomes the one which was written: a + # column of documents holds values, and None is the value pandas + # hands back for a cell parquet stored as null. + return None if not _is_stated(cell) else cell + try: + return json.loads(cell) + except ValueError as error: + msg = ( + f"The column {name!r} of {quote_path(path)} holds " + f"{cell!r}, which is not a JSON document: {error}." + ) + raise ParameterError(msg) from error + + +def _stated_metadata(metadata) -> dict[str, str]: + """Return the key-value metadata a file states, as text.""" + out = {} + for key, value in (metadata or {}).items(): + # Decoded leniently, keys as well as values: this is another + # writer's metadata as often as it is ours, and a key which is not + # UTF-8 is something to report rather than something to hide. + name = key.decode(errors="replace") + # What pyarrow writes for itself, which is not the caller's to read. + if name == "pandas" or name.startswith("ARROW:"): + continue + out[name] = value.decode(errors="replace") + return out + + def row_cells(row) -> dict[str, str]: """ Return a row's stated cells, an empty one meaning unset. diff --git a/dascore/utils/time.py b/dascore/utils/time.py index 336a91b81..08d5b3091 100644 --- a/dascore/utils/time.py +++ b/dascore/utils/time.py @@ -164,8 +164,17 @@ def _float_to_datetime(ser: pd.Series) -> pd.Series: @to_datetime64.register(pd.arrays.StringArray) +@to_datetime64.register(pd.arrays.ArrowStringArray) def _string_array_to_datetime64(arr: pd.arrays.StringArray): - """Convert pandas StringArray to datetime64.""" + """ + Convert a pandas string array to datetime64. + + Both backings, since which one pandas gives text is not the caller's + choice: a `str` column is arrow-backed wherever pyarrow is installed + and numpy-backed where it is not. Neither class is a subclass of the + other -- they share only `BaseStringArray` -- so registering one does + not dispatch the other. + """ out = pd.to_datetime(arr, errors="coerce", format="mixed") return out.to_numpy(dtype="datetime64[ns]") @@ -313,8 +322,9 @@ def _series_to_timedelta64_series(ser: pd.Series) -> pd.Series: @to_timedelta64.register(pd.arrays.StringArray) +@to_timedelta64.register(pd.arrays.ArrowStringArray) def _string_array_to_timedelta64(arr: pd.arrays.StringArray): - """Convert pandas StringArray to timedelta64.""" + """Convert a pandas string array, of either backing, to timedelta64.""" out = pd.to_timedelta(arr, errors="coerce") return out.to_numpy(dtype="timedelta64[ns]") diff --git a/environment.yml b/environment.yml index 6150b359c..8a32ae726 100644 --- a/environment.yml +++ b/environment.yml @@ -15,6 +15,7 @@ dependencies: - matplotlib>=3.5 - scipy>=1.15.0 - findiff + - pyarrow - jupyter - nbformat - pint diff --git a/pyproject.toml b/pyproject.toml index a465f7a68..04461b8be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,9 @@ dependencies = [ extras = [ "xarray", + # An annotation set may be stored as parquet rather than CSV; CSV is + # the floor which needs nothing. + "pyarrow", "netCDF4", "h5netcdf", "findiff", diff --git a/tests/test_core/test_annotation_loader.py b/tests/test_core/test_annotation_loader.py index c23e0a217..f8fea8bf4 100644 --- a/tests/test_core/test_annotation_loader.py +++ b/tests/test_core/test_annotation_loader.py @@ -3,6 +3,8 @@ from __future__ import annotations import json +import tempfile +from pathlib import Path import numpy as np import pandas as pd @@ -13,13 +15,55 @@ except ImportError: yaml = None +try: + import pyarrow + import pyarrow.parquet +except ImportError: + pyarrow = None + import dascore as dc -from dascore.core.annotations import Line, Moveout +from dascore.core.annotation_loader import find_annotations +from dascore.core.annotations import DIMS_KEY, Line, Moveout, _one_file from dascore.exceptions import InvalidAnnotationError, ParameterError +from dascore.utils.tables import DOCUMENT_KEY, write_parquet DIMS = ("distance", "time") +def _folds_case() -> bool: + """Return True if this filesystem holds two case variants as one file.""" + with tempfile.TemporaryDirectory() as name: + directory = Path(name) + (directory / "CaseProbe").write_text("") + return (directory / "caseprobe").exists() + + +# Asked once, at collection, as the inventory's tests ask it: Windows and +# most macOS checkouts fold case, so a directory named for another with a +# different case cannot exist there to be refused. +FOLDS_CASE = _folds_case() + + +def _denies_access() -> bool: + """Return True if a directory can be made unreadable by chmod.""" + with tempfile.TemporaryDirectory() as name: + directory = Path(name) / "locked" + directory.mkdir() + directory.chmod(0o000) + try: + list(directory.iterdir()) + return False + except OSError: + return True + finally: + directory.chmod(0o755) + + +# Windows keeps a directory listable whatever its mode, and root reads +# everything, so neither can be shown the failure this names. +DENIES_ACCESS = _denies_access() + + @pytest.fixture def curve() -> Moveout: """A moveout a path may be drawn from.""" @@ -91,6 +135,33 @@ def with_vertices(curve) -> dc.AnnotationSet: return dc.AnnotationSet(frame, dims=DIMS, vertices=vertices) +@pytest.fixture +def picks() -> dc.AnnotationSet: + """A set of time ranges made by a picker, on its own acquisition.""" + frame = pd.DataFrame( + { + "id": ["m1", "m2"], + "group": ["arrival", "arrival"], + "value": ["p", "s"], + "time_start": [ + np.datetime64("2020-01-01T00:00:01"), + np.datetime64("2020-01-01T00:00:03"), + ], + "time_end": [ + np.datetime64("2020-01-01T00:00:02"), + np.datetime64("2020-01-01T00:00:04"), + ], + "score": [0.4, 0.6], + } + ) + return dc.AnnotationSet( + frame, + dims=("time",), + acquisition_key="NET.ARR.00.fast", + creation_info={"author": "phasenet"}, + ) + + class TestRoundTrip: """A set written out and read back is the set it was.""" @@ -290,6 +361,15 @@ def test_a_shouted_suffix_is_read_and_superseded(self, regions, tmp_path): ) assert dc.annotations(directory) == regions + def test_one_file_under_two_names(self, tmp_path): + """The question the supersede pass asks; the test above is where it bites.""" + path = tmp_path / "attrs.json" + path.write_text("{}") + assert _one_file(path, tmp_path / "." / "attrs.json") + # A name nothing is written under reaches no file, so it is not + # the one just written and stays stale. + assert not _one_file(path, tmp_path / "attrs.yaml") + class TestTheDoor: """Everything a set may be loaded from goes through one function.""" @@ -308,9 +388,9 @@ def test_a_set_refuses_overrides(self, regions): def test_a_directory_refuses_what_it_states(self, regions, tmp_path): """A directory holds its own attributes and vertices.""" directory = regions.io.save(tmp_path / "picks") - with pytest.raises(InvalidAnnotationError, match="a set directory"): + with pytest.raises(InvalidAnnotationError, match="which states them"): dc.annotations(directory, attrs={"dims": DIMS}) - with pytest.raises(InvalidAnnotationError, match="a set directory"): + with pytest.raises(InvalidAnnotationError, match="which states them"): dc.annotations(directory, vertices=pd.DataFrame()) def test_a_dataframe(self): @@ -485,7 +565,7 @@ def test_no_annotations_table(self, tmp_path): """A directory without one states no annotations.""" directory = tmp_path / "picks" directory.mkdir() - with pytest.raises(InvalidAnnotationError, match=r"no annotations\.csv"): + with pytest.raises(InvalidAnnotationError, match="no annotations table"): dc.annotations(directory, dims=DIMS) def test_a_table_which_cannot_be_read(self, regions, tmp_path): @@ -495,6 +575,22 @@ def test_a_table_which_cannot_be_read(self, regions, tmp_path): with pytest.raises(InvalidAnnotationError, match="Could not read"): dc.annotations(directory) + def test_vertices_named_in_another_case(self, with_vertices, tmp_path): + """Every part of a set is found the same way, so none is skipped.""" + directory = with_vertices.io.save(tmp_path / "picks") + table = directory / "vertices.csv" + table.rename(directory / "vertices.CSV") + assert dc.annotations(directory) == with_vertices + + @pytest.mark.skipif(FOLDS_CASE, reason="this filesystem holds one of the two") + def test_vertices_spelled_twice(self, with_vertices, tmp_path): + """A set spells each of its parts once, vertices included.""" + directory = with_vertices.io.save(tmp_path / "picks") + text = (directory / "vertices.csv").read_text() + (directory / "vertices.CSV").write_text(text) + with pytest.raises(InvalidAnnotationError, match="states vertices more than"): + dc.annotations(directory) + def test_a_stray_table(self, regions, tmp_path): """A near-miss on the convention raises rather than being skipped.""" directory = regions.io.save(tmp_path / "picks") @@ -655,3 +751,932 @@ def test_the_attrs_name_their_model(self, regions, tmp_path): directory = regions.io.save(tmp_path / "picks") text = (directory / "attrs.json").read_text() assert '"object_type": "AnnotationSetAttrs"' in text + + +class TestCollections: + """Sets stored side by side read as one set which says where each row came from.""" + + @pytest.fixture + def collection(self, regions, picks, tmp_path): + """A directory holding two sets, each stating its own dimensions.""" + root = tmp_path / "sets" + regions.io.save(root / "hand") + picks.io.save(root / "phasenet") + return root + + def test_reads_as_one_set(self, collection, regions, picks): + """Every set is in the table, in the dimensions all of them state.""" + loaded = dc.annotations(collection) + assert len(loaded) == len(regions) + len(picks) + assert loaded.dims == ("distance", "time") + + def test_names_the_set_each_row_came_from(self, collection): + """The directory name is the label, in the set column and in attrs.sets.""" + loaded = dc.annotations(collection) + assert {x.set for x in loaded} == {"hand", "phasenet"} + assert sorted(loaded.attrs.sets) == ["hand", "phasenet"] + + def test_keeps_what_a_set_states_for_itself(self, collection, picks): + """A child's dimensions and provenance survive without filling rows.""" + stated = dc.annotations(collection).attrs.sets["phasenet"] + assert stated.dims == ("time",) + assert stated.creation_info.author == "phasenet" + assert stated.acquisition_key == picks.attrs.acquisition_key + + def test_a_row_keeps_its_own_acquisition(self, collection, regions, picks): + """The address a row was picked on is the address of its set.""" + loaded = dc.annotations(collection) + keys = {x.set: x.acquisition_key for x in loaded} + assert keys["hand"] == regions.attrs.acquisition_key + assert keys["phasenet"] == picks.attrs.acquisition_key + + def test_round_trips_through_one_directory(self, collection, tmp_path): + """A collection saves flat, and what it read is what it reads back.""" + loaded = dc.annotations(collection) + assert dc.annotations(loaded.io.save(tmp_path / "flat")) == loaded + + def test_an_id_in_two_sets(self, regions, tmp_path): + """An id is an address into the collection, so it names one row.""" + root = tmp_path / "sets" + regions.io.save(root / "hand") + regions.io.save(root / "again") + with pytest.raises(InvalidAnnotationError, match="again and hand"): + dc.annotations(root) + + def test_a_set_which_states_only_attributes(self, collection, tmp_path): + """A directory of attributes and nothing else is half a set.""" + half = collection / "empty" + half.mkdir() + (half / "attrs.json").write_text('{"dims": ["time"]}') + with pytest.raises( + InvalidAnnotationError, match="states the attributes of a set but no" + ): + dc.annotations(collection) + + def test_a_tree_which_holds_no_set_at_all(self, tmp_path): + """Nothing is refused until a directory turns out to be a collection.""" + root = tmp_path / "data" + half = root / "notes" + half.mkdir(parents=True) + (half / "attrs.json").write_text('{"dims": ["time"]}') + # The half-set is not what is wrong here: this directory states no + # annotations of its own, and holds no set to make it a collection. + with pytest.raises(InvalidAnnotationError, match="holds no annotations table"): + dc.annotations(root, dims=("time",)) + + def test_a_tree_of_collections(self, regions, tmp_path): + """Sets loaded together are one collection, not a tree of them.""" + root = tmp_path / "sets" + regions.io.save(root / "outer" / "inner") + regions.io.save(root / "hand") + with pytest.raises(InvalidAnnotationError, match="not a tree"): + dc.annotations(root) + + def test_a_set_which_also_holds_sets(self, regions, picks, tmp_path): + """A directory stating annotations is the set, whatever sits below it.""" + root = regions.io.save(tmp_path / "sets") + picks.io.save(root / "hand") + assert dc.annotations(root) == regions + + def test_a_set_beside_a_folder_of_its_own(self, regions, tmp_path): + """A folder someone kept beside the tables is not this format's business.""" + root = regions.io.save(tmp_path / "picks") + (root / "backup").mkdir() + (root / "backup" / "attrs.json").write_text('{"dims": ["time"]}') + assert dc.annotations(root) == regions + + def test_a_table_beside_the_sets(self, collection): + """A table where every set is a directory names no set.""" + (collection / "notes.csv").write_text("group\nnoise\n") + with pytest.raises(InvalidAnnotationError, match="name no set"): + dc.annotations(collection) + + def test_dimensions_for_the_sets_which_state_none(self, regions, tmp_path): + """They reach the children which declare none, and without them none do.""" + root = tmp_path / "sets" + for name in ("hand", "auto"): + directory = root / name + directory.mkdir(parents=True) + (directory / "annotations.csv").write_text( + f"id,group,time_start,time_end\n{name},noise,1.0,2.0\n" + ) + assert len(dc.annotations(root, dims=("time",))) == 2 + with pytest.raises(InvalidAnnotationError, match="states no dimensions"): + dc.annotations(root) + + def test_dimensions_stated_beside_the_sets(self, tmp_path): + """A collection may declare them, and then refuses a caller restating them.""" + root = tmp_path / "sets" + directory = root / "hand" + directory.mkdir(parents=True) + (directory / "annotations.csv").write_text("group,time_start,time_end\nq,1,2\n") + (root / "attrs.json").write_text('{"dims": ["time"]}') + assert dc.annotations(root).dims == ("time",) + with pytest.raises( + InvalidAnnotationError, match="a directory of sets stating its own" + ): + dc.annotations(root, dims=("time",)) + + def test_a_dimension_spelled_two_ways(self, picks, tmp_path): + """A point is not a range of no width, so neither stands in.""" + root = tmp_path / "sets" + picks.io.save(root / "ranges") + frame = pd.DataFrame({"group": ["a"], "time": [1.0]}) + dc.AnnotationSet(frame, dims=("time",)).io.save(root / "points") + with pytest.raises(InvalidAnnotationError, match="spelled as a point") as info: + dc.annotations(root) + # The constructor refuses this too; naming the sets is what this adds. + assert "points" in str(info.value) and "ranges" in str(info.value) + + def test_a_set_which_states_a_set_column(self, tmp_path): + """The set column names the set, so a set may not fill it in.""" + root = tmp_path / "sets" + directory = root / "hand" + directory.mkdir(parents=True) + (directory / "annotations.csv").write_text("set,group,time\nother,noise,1.0\n") + with pytest.raises(InvalidAnnotationError, match="states a set column"): + dc.annotations(root, dims=("time",)) + + def test_sets_stated_twice(self, collection): + """A collection states each of its sets once.""" + document = {"dims": ["time"], "sets": {"hand": {"dims": ["time"]}}} + (collection / "attrs.json").write_text(json.dumps(document)) + with pytest.raises( + InvalidAnnotationError, match="states each of its sets once" + ): + dc.annotations(collection) + + def test_hidden_directories_are_not_sets(self, collection, regions): + """A hidden name beside the sets describes the data, not them.""" + regions.io.save(collection / ".annotations") + assert sorted(dc.annotations(collection).attrs.sets) == ["hand", "phasenet"] + + def test_a_directory_which_is_no_set(self, collection): + """A directory participating in no convention here is left alone.""" + (collection / "figures").mkdir() + assert len(dc.annotations(collection)) == 4 + + def test_a_row_keeps_the_acquisition_it_names(self, regions, tmp_path): + """A row naming its own acquisition outranks its set's, merged or not.""" + root = tmp_path / "sets" + regions.io.save(root / "hand") + frame = pd.DataFrame( + { + "id": ["m1", "m2"], + "acquisition_key": ["NET.OTHER.00.das", None], + "time_start": [ + np.datetime64("2020-01-01T00:00:31"), + np.datetime64("2020-01-01T00:00:33"), + ], + "time_end": [ + np.datetime64("2020-01-01T00:00:32"), + np.datetime64("2020-01-01T00:00:34"), + ], + } + ) + other = dc.AnnotationSet( + frame, dims=("time",), acquisition_key="NET.SET.00.das" + ) + other.io.save(root / "auto") + keys = {x.id: x.acquisition_key for x in dc.annotations(root)} + assert keys["m1"] == "NET.OTHER.00.das" + assert keys["m2"] == "NET.SET.00.das" + assert keys["r1"] == regions.attrs.acquisition_key + + def test_what_the_collection_states_reaches_the_merged_set(self, tmp_path): + """A collection may state its own provenance beside its sets.""" + root = tmp_path / "sets" + directory = root / "hand" + directory.mkdir(parents=True) + (directory / "annotations.csv").write_text("group,time\nq,1\n") + document = { + "dims": ["time"], + "acquisition_key": "NET.COLL.00.das", + "history": ["decimate"], + } + (root / "attrs.json").write_text(json.dumps(document)) + loaded = dc.annotations(root) + assert loaded.attrs.history == ("decimate",) + # The row's own set states no key, so the collection's is its address. + assert loaded[0].acquisition_key == "NET.COLL.00.das" + + def test_dimensions_given_for_a_set_which_states_its_own(self, collection): + """Dropping the argument silently is the worse failure, here as anywhere.""" + with pytest.raises(InvalidAnnotationError, match="which states its own"): + dc.annotations(collection, dims=("distance", "time")) + + def test_a_dimension_stated_in_two_kinds(self, tmp_path): + """One set stating a time in seconds and another in dates cannot merge.""" + root = tmp_path / "sets" + for name, cell in (("clock", "2020-01-01T00:00:01"), ("numeric", "1.5")): + directory = root / name + directory.mkdir(parents=True) + (directory / "annotations.csv").write_text(f"group,time\nq,{cell}\n") + (directory / "attrs.json").write_text('{"dims": ["time"]}') + with pytest.raises(InvalidAnnotationError, match="different kinds of value"): + dc.annotations(root) + + def test_a_dimension_no_row_of_one_set_states(self, picks, tmp_path): + """A column every row leaves empty states no kind, so it agrees.""" + root = tmp_path / "sets" + picks.io.save(root / "phasenet") + blank = root / "quiet" + blank.mkdir(parents=True) + (blank / "annotations.csv").write_text("group,time_start,time_end\nq,,\n") + (blank / "attrs.json").write_text('{"dims": ["time"]}') + assert len(dc.annotations(root)) == len(picks) + 1 + + def test_a_column_which_is_a_dimension_in_one_set_only(self, tmp_path): + """A column another set dimensions must not silently become a bound.""" + root = tmp_path / "sets" + notes = root / "notes" + notes.mkdir(parents=True) + (notes / "annotations.csv").write_text("group,time,distance\nq,1,shallow\n") + (notes / "attrs.json").write_text('{"dims": ["time"]}') + boxes = root / "boxes" + boxes.mkdir(parents=True) + (boxes / "annotations.csv").write_text("group,time,distance\nb,2,50\n") + (boxes / "attrs.json").write_text('{"dims": ["time", "distance"]}') + with pytest.raises(InvalidAnnotationError, match="without declaring"): + dc.annotations(root) + + @pytest.mark.skipif(FOLDS_CASE, reason="this filesystem holds one of the two") + def test_set_names_which_differ_only_in_case(self, regions, picks, tmp_path): + """A set name is a label, so it must name one set on any filesystem.""" + root = tmp_path / "sets" + regions.io.save(root / "hand") + picks.io.save(root / "HAND") + with pytest.raises(InvalidAnnotationError, match="differ only in case"): + dc.annotations(root) + + def test_a_hidden_table_beside_the_sets(self, collection): + """A half-copied file is a companion, not a table which names no set.""" + (collection / ".annotations.csv").write_text("group\nnoise\n") + assert len(dc.annotations(collection)) == 4 + + def test_a_stray_table_whatever_its_case(self, collection): + """The suffix is matched as the loader matches every other one.""" + (collection / "NOTES.CSV").write_text("group\nnoise\n") + with pytest.raises(InvalidAnnotationError, match=r"NOTES\.CSV"): + dc.annotations(collection) + + def test_a_set_named_in_upper_case(self, tmp_path): + """A set states its table once, in whichever case it spells the suffix.""" + directory = tmp_path / "sets" / "hand" + directory.mkdir(parents=True) + (directory / "annotations.CSV").write_text("group,time\nq,1\n") + assert len(dc.annotations(tmp_path / "sets", dims=("time",))) == 1 + + def test_a_directory_of_other_things(self, collection): + """A directory which states no annotations is left alone, empty or not.""" + figures = collection / "figures" + figures.mkdir() + (figures / "map.png").write_bytes(b"not an image either") + (figures / "notes.txt").write_text("nothing to do with the format") + assert len(dc.annotations(collection)) == 4 + + def test_a_collection_saved_flat_is_not_a_member(self, collection, tmp_path): + """A directory this library wrote is named for what it is.""" + root = tmp_path / "outer" + dc.annotations(collection).io.save(root / "merged") + with pytest.raises(InvalidAnnotationError, match="already a collection"): + dc.annotations(root) + + @pytest.mark.skipif(not DENIES_ACCESS, reason="a mode cannot deny a read here") + def test_a_directory_which_cannot_be_read(self, collection): + """A tightened permission is named as an annotation error, not an OSError.""" + locked = collection / "locked" + locked.mkdir() + locked.chmod(0o000) + try: + with pytest.raises(InvalidAnnotationError, match="Could not read"): + dc.annotations(collection) + finally: + locked.chmod(0o755) + + def test_a_label_naming_no_set(self, collection, tmp_path): + """A label reaches back to what its set says, so it names one.""" + flat = dc.annotations(collection).io.save(tmp_path / "flat") + table = flat / "annotations.csv" + table.write_text(table.read_text().replace(",hand", ",typo")) + with pytest.raises(InvalidAnnotationError, match="name no set stated here"): + dc.annotations(flat) + + def test_a_row_with_no_label(self, collection, tmp_path): + """A row loaded with others says which of them it came from.""" + flat = dc.annotations(collection).io.save(tmp_path / "flat") + table = flat / "annotations.csv" + text = table.read_text().replace(",hand", ",", 1) + table.write_text(text) + with pytest.raises(InvalidAnnotationError, match="state no set"): + dc.annotations(flat) + + def test_a_table_with_no_label_column(self, collection, tmp_path): + """Sets stated with no column to name them leave every row adrift.""" + flat = dc.annotations(collection).io.save(tmp_path / "flat") + table = flat / "annotations.csv" + frame = dc.annotations(flat).io.to_dataframe().drop(columns="set") + table.write_text(frame.to_csv(index=False)) + with pytest.raises(InvalidAnnotationError, match="no set column"): + dc.annotations(flat) + + def test_a_collection_of_empty_sets(self, tmp_path): + """A collection holding no rows at all labels none of them.""" + root = tmp_path / "sets" + for name in ("hand", "auto"): + dc.AnnotationSet(None, dims=("time",)).io.save(root / name) + loaded = dc.annotations(root) + assert len(loaded) == 0 + assert sorted(loaded.attrs.sets) == ["auto", "hand"] + + def test_a_set_column_on_a_set_of_its_own(self): + """A set which states no sets is not a collection, so its labels are its own.""" + frame = pd.DataFrame({"set": ["whatever"], "group": ["a"], "time": [1.0]}) + assert dc.AnnotationSet(frame, dims=("time",))[0].set == "whatever" + + def test_a_set_with_no_annotations(self, picks, tmp_path): + """A set which states nothing is still one of the sets loaded.""" + root = tmp_path / "sets" + picks.io.save(root / "phasenet") + dc.AnnotationSet(None, dims=("time",)).io.save(root / "empty") + loaded = dc.annotations(root) + assert len(loaded) == len(picks) + assert sorted(loaded.attrs.sets) == ["empty", "phasenet"] + + def test_paths_from_two_sets(self, with_vertices, tmp_path): + """Vertices merge too, and each path still reads as the shape it was.""" + root = tmp_path / "sets" + with_vertices.io.save(root / "hand") + frame = pd.DataFrame({"id": ["p9"], "group": ["auto"], "geometry": ["path"]}) + vertices = pd.DataFrame( + { + "id": ["p9"] * 3, + "seq": [0, 1, 2], + "distance": [1000.0, 1100.0, 1200.0], + "time": np.array( + [ + "2020-01-01T00:00:05", + "2020-01-01T00:00:06", + "2020-01-01T00:00:07", + ], + dtype="datetime64[ns]", + ), + } + ) + dc.AnnotationSet(frame, dims=DIMS, vertices=vertices).io.save(root / "auto") + drawn = {x.id: x.geometry for x in dc.annotations(root) if x.id.startswith("p")} + assert drawn["p1"].vertices["distance"] == (10.0, 95.0, 185.0) + assert drawn["p9"].vertices["distance"] == (1000.0, 1100.0, 1200.0) + assert drawn["p9"].region.bounds["distance"] == (1000.0, 1200.0) + + def test_vertices_in_different_dimensions(self, with_vertices, tmp_path): + """A vertex states every dimension its table names, so these cannot merge.""" + root = tmp_path / "sets" + with_vertices.io.save(root / "hand") + frame = pd.DataFrame({"id": ["f1"], "geometry": ["path"]}) + vertices = pd.DataFrame( + {"id": ["f1"] * 2, "seq": [0, 1], "distance": [1.0, 2.0]} + ) + dc.AnnotationSet(frame, dims=("distance",), vertices=vertices).io.save( + root / "flat" + ) + with pytest.raises( + InvalidAnnotationError, match="different dimensions" + ) as info: + dc.annotations(root) + assert "hand" in str(info.value) and "flat" in str(info.value) + + +class TestDeclaringDimensionsInTheTable: + """A bare table has no attrs file, so it may declare them above its header.""" + + def test_a_bare_table_declares_them(self, tmp_path): + """The dimensions travel with the file rather than with the call.""" + path = tmp_path / "picks.csv" + path.write_text("# dims: distance, time\ngroup,time_start,time_end\nq,1,2\n") + loaded = dc.annotations(path) + assert loaded.dims == ("distance", "time") + # The header is the one below the pragma, not the pragma itself. + assert set(loaded.io.to_dataframe().columns) == { + "group", + "time_start", + "time_end", + } + assert loaded[0].group == "q" + assert loaded[0].region.bounds["time"] == (1.0, 2.0) + + def test_other_comments_are_comments(self, tmp_path): + """A line above the header which declares nothing says nothing.""" + path = tmp_path / "picks.csv" + path.write_text("# picked by hand\n#dims:time\ngroup,time\nq,1\n") + loaded = dc.annotations(path) + assert loaded.dims == ("time",) + assert loaded[0].region.bounds["time"] == (1.0, 1.0) + + def test_restating_them_is_allowed(self, tmp_path): + """Two spellings of one fact agree or they are not one fact.""" + path = tmp_path / "picks.csv" + path.write_text("# dims: time\ngroup,time\nq,1\n") + loaded = dc.annotations(path, dims=("time",)) + assert loaded.dims == ("time",) + assert loaded[0].region.bounds["time"] == (1.0, 1.0) + + def test_disagreeing_with_the_caller(self, tmp_path): + """The table's dimensions and the caller's are not merged.""" + path = tmp_path / "picks.csv" + path.write_text("# dims: time\ngroup,time\nq,1\n") + with pytest.raises(InvalidAnnotationError, match="where the two agree"): + dc.annotations(path, dims=("distance",)) + + def test_a_header_which_starts_with_the_mark(self, tmp_path): + """A column may be named `#note`, and that line is the header.""" + path = tmp_path / "picks.csv" + path.write_text("#note,group,time\nfirst,a,1\nsecond,b,2\n") + loaded = dc.annotations(path, dims=("time",)) + assert len(loaded) == 2 + assert loaded[0].extra["#note"] == "first" + + def test_a_declaration_commented_out(self, tmp_path): + """A struck-out declaration declares nothing, so the line is a header.""" + path = tmp_path / "picks.csv" + path.write_text("## dims: time\ngroup,time\nq,1\n") + with pytest.raises(InvalidAnnotationError, match="cells where its header"): + dc.annotations(path, dims=("time",)) + + def test_a_header_which_reads_as_a_comment(self, tmp_path): + """A column may be named `# note`, which this library writes unquoted.""" + frame = pd.DataFrame( + {"# note": ["first", "second"], "group": ["a", "b"], "time": [1.0, 2.0]} + ) + picks = dc.AnnotationSet(frame, dims=("time",)) + path = tmp_path / "picks.csv" + picks.io.to_csv(path) + loaded = dc.annotations(path, dims=("time",)) + assert len(loaded) == 2 + assert loaded[0].extra["# note"] == "first" + + def test_comments_beside_a_declaration(self, tmp_path): + """Where a table declares its dimensions, comments ride with it.""" + path = tmp_path / "picks.csv" + path.write_text("# dims: time\n# picked by hand\ngroup,time\nq,1\n") + loaded = dc.annotations(path) + assert loaded.dims == ("time",) + assert set(loaded.io.to_dataframe().columns) == {"group", "time"} + + def test_the_keyword_is_read_in_any_case(self, tmp_path): + """A hand-authored line is read as written, whatever case it names.""" + path = tmp_path / "picks.csv" + path.write_text("# Dims: time , distance\ngroup,time\nq,1\n") + assert dc.annotations(path).dims == ("time", "distance") + + def test_a_blank_line_above_the_declaration(self, tmp_path): + """Blank lines above the header are skipped with the comments.""" + path = tmp_path / "picks.csv" + path.write_text("\n# dims: time\n\ngroup,time\nq,1\n") + assert dc.annotations(path).dims == ("time",) + + def test_a_table_which_cannot_be_decoded(self, tmp_path): + """A table which cannot be read has no dimensions to be found in it.""" + path = tmp_path / "picks.csv" + path.write_bytes(b"# dims: time\ngroup,time\n\xff\xfe,1\n") + with pytest.raises(InvalidAnnotationError, match="Could not read"): + dc.annotations(path) + + def test_a_comment_holding_a_quote(self, tmp_path): + """A comment is one line, whatever a csv reader makes of its quotes.""" + path = tmp_path / "picks.csv" + path.write_text('# dims: time\n# it is ,"odd\ngroup,time\nq,1\n') + loaded = dc.annotations(path) + assert loaded.dims == ("time",) + assert set(loaded.io.to_dataframe().columns) == {"group", "time"} + + def test_disagreeing_with_what_the_attrs_state(self, regions, tmp_path): + """The message says where the other spelling came from.""" + directory = regions.io.save(tmp_path / "picks") + table = directory / "annotations.csv" + table.write_text("# dims: depth\n" + table.read_text()) + with pytest.raises(InvalidAnnotationError, match="is stated in its attributes"): + dc.annotations(directory) + + def test_a_child_declaring_its_own_above_its_table(self, tmp_path): + """A pragma is a set stating its dimensions, as its attrs would be.""" + root = tmp_path / "sets" + for name, dim in (("hand", "time"), ("auto", "distance")): + directory = root / name + directory.mkdir(parents=True) + (directory / "annotations.csv").write_text( + f"# dims: {dim}\ngroup,{dim}\nq,1\n" + ) + (root / "attrs.json").write_text('{"dims": ["time"]}') + with pytest.raises(InvalidAnnotationError, match="which states its own"): + dc.annotations(root) + + def test_disagreeing_with_the_attrs(self, regions, tmp_path): + """A set directory states them once, wherever it states them.""" + directory = regions.io.save(tmp_path / "picks") + table = directory / "annotations.csv" + table.write_text("# dims: depth\n" + table.read_text()) + with pytest.raises(InvalidAnnotationError, match="where the two agree"): + dc.annotations(directory) + + def test_a_set_directory_may_declare_them(self, tmp_path): + """A hand-made set directory need not carry an attrs file.""" + directory = tmp_path / "picks" + directory.mkdir() + (directory / "annotations.csv").write_text("# dims: time\ngroup,time\nq,1\n") + loaded = dc.annotations(directory) + assert loaded.dims == ("time",) + assert loaded[0].region.bounds["time"] == (1.0, 1.0) + + def test_sets_loaded_together_may_declare_them(self, tmp_path): + """Each set in a collection may state its own, above its own table.""" + root = tmp_path / "sets" + for name, dim in (("hand", "time"), ("auto", "distance")): + directory = root / name + directory.mkdir(parents=True) + (directory / "annotations.csv").write_text( + f"# dims: {dim}\ngroup,{dim}\nq,1\n" + ) + loaded = dc.annotations(root) + assert loaded.dims == ("distance", "time") + bounds = sorted(tuple(x.region.bounds.items()) for x in loaded) + assert bounds == [ + (("distance", (1.0, 1.0)),), + (("time", (1.0, 1.0)),), + ] + + def test_declared_twice(self, tmp_path): + """One table states its dimensions once.""" + path = tmp_path / "picks.csv" + path.write_text("# dims: time\n# dims: distance\ngroup,time\nq,1\n") + with pytest.raises(InvalidAnnotationError, match="more than once"): + dc.annotations(path) + + def test_declared_but_named_none(self, tmp_path): + """A declaration which names nothing declares nothing.""" + path = tmp_path / "picks.csv" + path.write_text("# dims:\ngroup,time\nq,1\n") + with pytest.raises(InvalidAnnotationError, match="names none"): + dc.annotations(path) + + def test_vertices_declare_nothing(self, with_vertices, tmp_path): + """Vertices are read in the dimensions of the set they belong to.""" + directory = with_vertices.io.save(tmp_path / "picks") + table = directory / "vertices.csv" + table.write_text("# dims: time\n" + table.read_text()) + with pytest.raises(InvalidAnnotationError, match="states them once"): + dc.annotations(directory) + + def test_vertices_take_no_preamble(self, with_vertices, tmp_path): + """Vertices declare nothing, so they have nothing to comment beside.""" + directory = with_vertices.io.save(tmp_path / "picks") + table = directory / "vertices.csv" + table.write_text("# drawn on a screen\n" + table.read_text()) + with pytest.raises(InvalidAnnotationError, match="cells where its header"): + dc.annotations(directory) + + +class TestCarriedAnnotations: + """A directory of data carries what it was annotated with, hidden beside it.""" + + @pytest.fixture + def data(self, tmp_path): + """A directory of data, with nothing of this format visible in it.""" + directory = tmp_path / "data" + directory.mkdir() + (directory / "das_1.h5").write_text("pretend this is data") + return directory + + def test_a_carried_set(self, data, regions): + """Loading a directory of data loads the annotations it carries.""" + regions.io.save(data / ".annotations") + assert dc.annotations(data) == regions + + def test_a_carried_collection(self, data, regions, picks): + """What is carried may be many named sets, as anywhere else.""" + regions.io.save(data / ".annotations" / "hand") + picks.io.save(data / ".annotations" / "phasenet") + assert sorted(dc.annotations(data).attrs.sets) == ["hand", "phasenet"] + + def test_a_carried_table(self, data): + """The bare table spelling is carried under the same name.""" + (data / ".annotations.csv").write_text("# dims: time\ngroup,time\nq,1\n") + assert dc.annotations(data).dims == ("time",) + + def test_a_carried_table_takes_what_it_does_not_state(self, data): + """A bare table states no attributes, so the caller may state them.""" + (data / ".annotations.csv").write_text("group,time\nq,1\n") + loaded = dc.annotations(data, attrs={"dims": ("time",), "history": ("de",)}) + assert loaded.dims == ("time",) + assert loaded.attrs.history == ("de",) + + def test_a_carried_set_directory_states_its_own(self, data, regions): + """A carried directory holds its attributes, as any set directory does.""" + regions.io.save(data / ".annotations") + with pytest.raises(InvalidAnnotationError, match="which states them"): + dc.annotations(data, attrs={"dims": DIMS}) + + def test_carried_twice(self, data, regions): + """A directory states what it carries once.""" + regions.io.save(data / ".annotations") + (data / ".annotations.csv").write_text("group,time\nq,1\n") + with pytest.raises(InvalidAnnotationError, match="more than once"): + dc.annotations(data) + + def test_the_wrong_kind_of_thing(self, data): + """Something under the blessed name in a form it does not take.""" + (data / ".annotations").write_text("not a set") + with pytest.raises(InvalidAnnotationError, match="is a file"): + dc.annotations(data) + + def test_a_table_name_holding_a_directory(self, data): + """The csv spelling is a table; a directory is the other one.""" + (data / ".annotations.csv").mkdir() + with pytest.raises(InvalidAnnotationError, match="is a directory"): + dc.annotations(data) + + def test_a_visible_set_is_the_set(self, data, regions, picks): + """A directory stating annotations is a set, not something carrying one.""" + picks.io.save(data / ".annotations") + regions.io.save(data) + assert dc.annotations(data) == regions + + def test_carrying_nothing(self, data): + """A directory with no annotations says so, and names the convention.""" + with pytest.raises(InvalidAnnotationError, match=r"\.annotations"): + dc.annotations(data, dims=DIMS) + + def test_find_annotations_judges_only_the_name(self, data): + """The path comes back because of its name, not because it loads.""" + assert find_annotations(data) is None + table = data / ".annotations.csv" + table.write_text("this is not a table at all") + assert find_annotations(data) == table + + def test_the_data_directory_keeps_its_own_attrs(self, data, regions): + """A data directory's attrs file is about the data, so it is not read.""" + (data / "attrs.json").write_text('{"object_type": "SomethingElse"}') + regions.io.save(data / ".annotations") + assert dc.annotations(data) == regions + + +def _forge(frame: pd.DataFrame, path, documents: str) -> None: + """Write a parquet file whose document footer this library did not write.""" + table = pyarrow.Table.from_pandas(frame, preserve_index=False) + kept = {**(table.schema.metadata or {}), DOCUMENT_KEY: documents} + pyarrow.parquet.write_table(table.replace_schema_metadata(kept), path) + + +@pytest.mark.skipif(pyarrow is None, reason="pyarrow is not installed") +class TestParquet: + """The same tables, with their types kept, for a set too big to want text.""" + + @pytest.fixture + def mixed(self) -> dc.AnnotationSet: + """A set whose columns hold what a CSV would have to spell as text.""" + frame = pd.DataFrame( + { + "id": ["r1", "r2"], + "group": ["noise", "quiet"], + "value": ["car", True], + "tags": [("road", "car"), None], + "time_start": [ + np.datetime64("2020-01-01T00:00:10"), + np.datetime64("2020-01-01T00:00:20"), + ], + "time_end": [ + np.datetime64("2020-01-01T00:00:12"), + np.datetime64("2020-01-01T00:00:22"), + ], + "score": [0.9, 0.2], + "checked": [True, False], + "meta": [{"a": 1}, None], + } + ) + return dc.AnnotationSet(frame, dims=DIMS, acquisition_key="NET.ARR.00.das") + + def test_a_bare_table(self, mixed, tmp_path): + """A set of regions is one file, and reads back as the set it was.""" + loaded = dc.annotations(mixed.io.to_parquet(tmp_path / "picks.parquet")) + assert loaded.io.to_dataframe().equals(mixed.io.to_dataframe()) + + def test_the_dimensions_travel_with_the_file(self, mixed, tmp_path): + """A parquet file states its dimensions where it can: its footer.""" + path = mixed.io.to_parquet(tmp_path / "picks.parquet") + assert dc.annotations(path).dims == DIMS + + def test_restating_the_dimensions(self, mixed, tmp_path): + """Agreement is allowed, disagreement is not, as with every spelling.""" + path = mixed.io.to_parquet(tmp_path / "picks.parquet") + assert dc.annotations(path, dims=DIMS).dims == DIMS + with pytest.raises(InvalidAnnotationError, match="where the two agree"): + dc.annotations(path, dims=("depth",)) + + def test_kinds_a_csv_would_lose(self, mixed, tmp_path): + """A column with no one type is written as documents, not as text.""" + loaded = dc.annotations(mixed.io.to_parquet(tmp_path / "picks.parquet")) + assert [type(x).__name__ for x in loaded.io.to_dataframe()["value"]] == [ + "str", + "bool", + ] + assert loaded[0].extra["meta"] == {"a": 1} + assert loaded[0].tags == ("road", "car") + + def test_text_stays_text(self, tmp_path): + """A typed format has a boolean, so a cell reading 'true' is the word.""" + frame = pd.DataFrame({"group": ["a"], "note": ["true"], "time": [1.0]}) + picks = dc.AnnotationSet(frame, dims=("time",)) + loaded = dc.annotations(picks.io.to_parquet(tmp_path / "picks.parquet")) + assert loaded[0].extra["note"] == "true" + + def test_a_value_a_csv_would_refuse(self, tmp_path): + """Text a table would read back as a boolean is safe where types are kept.""" + frame = pd.DataFrame({"group": ["a"], "value": ["true"], "time": [1.0]}) + picks = dc.AnnotationSet(frame, dims=("time",)) + with pytest.raises(ParameterError, match="a table would read back"): + picks.io.to_csv() + loaded = dc.annotations(picks.io.to_parquet(tmp_path / "picks.parquet")) + assert loaded[0].value == "true" + + def test_a_set_of_no_annotations(self, tmp_path): + """A set which states nothing writes a table which states nothing.""" + empty = dc.AnnotationSet(None, dims=("time",)) + assert ( + dc.annotations(empty.io.save(tmp_path / "picks", format="parquet")) == empty + ) + loaded = dc.annotations(empty.io.to_parquet(tmp_path / "picks.parquet")) + assert len(loaded) == 0 + + def test_an_empty_set_beside_a_full_one(self, picks, tmp_path): + """One set holding nothing does not take its collection down with it.""" + root = tmp_path / "sets" + picks.io.save(root / "phasenet", format="parquet") + dc.AnnotationSet(None, dims=("time",)).io.save(root / "empty", format="parquet") + assert len(dc.annotations(root)) == len(picks) + + def test_numbers_numpy_made(self, tmp_path): + """A value from numpy is the number it is, not the text str() gives.""" + frame = pd.DataFrame( + { + "group": ["g", "g", "h"], + "value": [np.int64(3), 5, "text"], + "time": [1.0, 2.0, 3.0], + } + ) + picks = dc.AnnotationSet(frame, dims=("time",)) + loaded = dc.annotations(picks.io.save(tmp_path / "picks", format="parquet")) + assert [x.value for x in loaded] == [3, 5, "text"] + + def test_a_time_inside_a_document(self, tmp_path): + """A time has no JSON type, so it is spelled as DASCore spells one.""" + frame = pd.DataFrame( + { + "group": ["a", "b"], + "meta": [{"when": np.datetime64("2020-01-01T00:00:01")}, "note"], + "time": [1.0, 2.0], + } + ) + picks = dc.AnnotationSet(frame, dims=("time",)) + loaded = dc.annotations(picks.io.save(tmp_path / "picks", format="parquet")) + assert loaded[0].extra["meta"] == {"when": "2020-01-01T00:00:01.000000000"} + + @pytest.mark.parametrize( + ("stated", "message"), + [(json.dumps("group"), "it is a list of names"), ("{oops", "not a JSON")], + ) + def test_a_footer_this_cannot_read(self, stated, message, tmp_path): + """A footer another writer left is read, or named where it cannot be.""" + path = tmp_path / "picks.parquet" + _forge(pd.DataFrame({"group": ["a"], "time": [1.0]}), path, stated) + with pytest.raises(InvalidAnnotationError, match=message): + dc.annotations(path, dims=("time",)) + + def test_a_directory(self, with_vertices, curve, tmp_path): + """Every part a set states is written under its own name.""" + directory = with_vertices.io.save(tmp_path / "picks", format="parquet") + assert sorted(x.name for x in directory.iterdir()) == [ + "annotations.parquet", + "attrs.json", + "vertices.parquet", + ] + loaded = dc.annotations(directory) + assert loaded == with_vertices + assert loaded[1].geometry.basis == curve + + def test_a_collection(self, regions, picks, tmp_path): + """A set is a set whichever encoding it is written in.""" + root = tmp_path / "sets" + regions.io.save(root / "hand", format="parquet") + picks.io.save(root / "phasenet") + assert sorted(dc.annotations(root).attrs.sets) == ["hand", "phasenet"] + + def test_carried_beside_data(self, regions, tmp_path): + """The hidden name takes the parquet spelling too.""" + directory = tmp_path / "data" + directory.mkdir() + regions.io.to_parquet(directory / ".annotations.parquet") + assert ( + dc.annotations(directory) + .io.to_dataframe() + .equals(regions.io.to_dataframe()) + ) + + def test_carried_whatever_case_it_names(self, regions, tmp_path): + """The carried name is matched as every other table name is.""" + directory = tmp_path / "data" + directory.mkdir() + regions.io.to_parquet(directory / ".annotations.PARQUET") + assert ( + dc.annotations(directory) + .io.to_dataframe() + .equals(regions.io.to_dataframe()) + ) + + def test_a_typed_column_which_cannot_be_a_dimension(self, tmp_path): + """Stating a type is not stating one a coordinate can be.""" + path = tmp_path / "picks.parquet" + write_parquet(pd.DataFrame({"group": ["a"], "time": [True]}), path) + with pytest.raises(InvalidAnnotationError, match="where it states numbers"): + dc.annotations(path, dims=("time",)) + + def test_a_typed_vertex_order_which_is_not_a_number(self, with_vertices, tmp_path): + """A vertex states its place in the order as a number, typed or not.""" + directory = with_vertices.io.save(tmp_path / "picks", format="parquet") + vertices = with_vertices.io.to_vertices() + stamps = np.array(["2020-01-01", "2020-01-02"], dtype="datetime64[ns]") + vertices["seq"] = list(stamps) * (len(vertices) // 2) + list( + stamps[: len(vertices) % 2] + ) + write_parquet(vertices, directory / "vertices.parquet") + with pytest.raises(InvalidAnnotationError, match="where it states a number"): + dc.annotations(directory) + + def test_the_other_encoding_is_superseded(self, regions, tmp_path): + """A set written twice states itself once, not once per encoding.""" + directory = regions.io.save(tmp_path / "picks") + assert (directory / "annotations.csv").exists() + regions.io.save(directory, format="parquet") + assert not (directory / "annotations.csv").exists() + assert dc.annotations(directory) == regions + regions.io.save(directory) + assert not (directory / "annotations.parquet").exists() + + def test_both_encodings_at_once(self, regions, tmp_path): + """A directory holding both says two things; neither is chosen.""" + directory = regions.io.save(tmp_path / "picks") + regions.io.to_parquet(directory / "annotations.parquet") + with pytest.raises(InvalidAnnotationError, match="each of its parts once"): + dc.annotations(directory) + + def test_a_bare_table_refuses_vertices(self, with_vertices, tmp_path): + """One file states one grain, whatever its encoding.""" + with pytest.raises(ParameterError, match="a bare table has no row for"): + with_vertices.io.to_parquet(tmp_path / "picks.parquet") + + def test_an_unknown_encoding(self, regions, tmp_path): + """A set is written in an encoding it has, and says which it has.""" + with pytest.raises(ParameterError, match="not a table encoding"): + regions.io.save(tmp_path / "picks", format="feather") + + def test_vertices_declare_nothing(self, with_vertices, tmp_path): + """Vertices are read in the dimensions of the set they belong to.""" + directory = with_vertices.io.save(tmp_path / "picks", format="parquet") + frame = with_vertices.io.to_vertices() + write_parquet(frame, directory / "vertices.parquet", {DIMS_KEY: '["time"]'}) + with pytest.raises(InvalidAnnotationError, match="states them once"): + dc.annotations(directory) + + def test_dimensions_which_are_not_a_document(self, regions, tmp_path): + """What the footer states is read, and named where it is not readable.""" + path = tmp_path / "picks.parquet" + write_parquet(regions.io.to_dataframe(), path, {DIMS_KEY: "time, distance"}) + with pytest.raises(InvalidAnnotationError, match="not a JSON document"): + dc.annotations(path) + + def test_dimensions_which_name_none(self, regions, tmp_path): + """A file which declares its dimensions names them.""" + path = tmp_path / "picks.parquet" + write_parquet(regions.io.to_dataframe(), path, {DIMS_KEY: "[]"}) + with pytest.raises(InvalidAnnotationError, match="names none"): + dc.annotations(path) + + def test_a_stray_parquet_table(self, regions, tmp_path): + """A near-miss on the convention is a near-miss in either encoding.""" + directory = regions.io.save(tmp_path / "picks") + regions.io.to_parquet(directory / "annotation.parquet") + with pytest.raises(InvalidAnnotationError, match=r"annotation\.parquet"): + dc.annotations(directory) + + def test_a_file_which_is_not_parquet(self, tmp_path): + """Whatever pyarrow makes of it, the error names the file.""" + path = tmp_path / "picks.parquet" + path.write_text("group,time\na,1\n") + with pytest.raises(InvalidAnnotationError, match="Could not read"): + dc.annotations(path, dims=("time",)) + + def test_a_table_which_is_neither(self, tmp_path): + """A bare set is a table, and the message names the encodings it takes.""" + path = tmp_path / "picks.txt" + path.write_text("group\na\n") + with pytest.raises(InvalidAnnotationError, match=r"\.csv or \.parquet"): + dc.annotations(path, dims=("time",)) diff --git a/tests/test_core/test_annotations.py b/tests/test_core/test_annotations.py index 48426a78f..3755260f7 100644 --- a/tests/test_core/test_annotations.py +++ b/tests/test_core/test_annotations.py @@ -245,6 +245,17 @@ def test_lone_range_column_is_an_extra(self): out = AnnotationSet(pd.DataFrame({"depth_start": [1]}), dims=DIMS) assert out[0].extra["depth_start"] == 1 + def test_the_set_column_is_a_label(self): + """A row read with others says which set it came from.""" + out = AnnotationSet(pd.DataFrame({"set": ["picks"]}), dims=DIMS) + assert out[0].set == "picks" + assert "set" not in out[0].extra + + def test_no_set_column_is_no_label(self): + """A set read on its own is not in a collection, so it names none.""" + out = AnnotationSet(pd.DataFrame({"group": ["a"]}), dims=DIMS) + assert out[0].set == "" + def test_declared_column_documents_only(self): """Documenting a column does not gate any other one.""" out = AnnotationSet( @@ -782,6 +793,17 @@ def test_attrs_are_frozen(self): with pytest.raises(ValidationError): AnnotationSetAttrs(dims=DIMS).dims = ("other",) + def test_sets_are_one_level_deep(self): + """Sets loaded together are one collection, not a tree of them.""" + child = AnnotationSetAttrs(dims=("time",), sets={"deeper": {"dims": ("time",)}}) + with pytest.raises(ValidationError, match="not a tree"): + AnnotationSetAttrs(dims=DIMS, sets={"picks": child}) + + def test_a_child_dimension_nothing_holds(self): + """A set states the dimensions the sets loaded with it are read in.""" + with pytest.raises(ValidationError, match="which the sets loaded with it"): + AnnotationSetAttrs(dims=("time",), sets={"picks": {"dims": ("depth",)}}) + class TestBasis: """Curves regenerate vertices; they are not geometries themselves.""" diff --git a/tests/test_utils/test_pd.py b/tests/test_utils/test_pd.py index 1c868d341..d6795dd94 100644 --- a/tests/test_utils/test_pd.py +++ b/tests/test_utils/test_pd.py @@ -22,6 +22,11 @@ ) from dascore.utils.time import to_datetime64, to_timedelta64 +try: + import pyarrow +except ImportError: + pyarrow = None + @pytest.fixture() def random_df_from_patch(random_patch): @@ -313,6 +318,13 @@ def test_ellipsis_kept_for_non_interval_column(self, example_df_2): out = filter_df(example_df_2, first_name=("Jason", ...)) assert np.all(out == example_df_2["first_name"].isin(["Jason"])) + @pytest.mark.skipif(pyarrow is None, reason="pyarrow is not installed") + def test_ellipsis_with_an_arrow_backed_column(self, example_df_2): + """An arrow-backed column refuses a value arrow has no type for.""" + df = example_df_2.astype({"first_name": "string[pyarrow]"}) + out = filter_df(df, first_name=("Jason", ...)) + assert np.all(out == df["first_name"].isin(["Jason"])) + def test_open_bound_ignored_for_unknown_column(self, example_df_2): """ Unknown columns are forwarded to patch level select, so an open bound diff --git a/tests/test_utils/test_tables.py b/tests/test_utils/test_tables.py index 5c24eba9f..21e8d8a2c 100644 --- a/tests/test_utils/test_tables.py +++ b/tests/test_utils/test_tables.py @@ -4,18 +4,32 @@ import csv +import numpy as np import pandas as pd import pytest +from dascore.core.annotations import Line from dascore.exceptions import ParameterError from dascore.utils.tables import ( + DOCUMENT_KEY, ordered_rows, + parquet_table, parse_cell, + read_parquet, + read_parquet_metadata, read_table, require_columns, require_stated, row_cells, + write_parquet, ) +from dascore.utils.time import to_datetime64, to_timedelta64 + +try: + import pyarrow + import pyarrow.parquet +except ImportError: + pyarrow = None def _write(path, text: str, name: str = "table.csv", encoding: str = "utf-8"): @@ -89,6 +103,18 @@ def test_oversized_cell_refused(self, tmp_path): with pytest.raises(ParameterError, match="field larger than field limit"): read_table(path) + def test_skipped_lines_are_not_the_header(self, tmp_path): + """A format may state something of its own above its table.""" + path = _write(tmp_path, "# dims: time\na,b\n1,2\n") + frame = read_table(path, skip=1) + assert list(frame.columns) == ["a", "b"] + + def test_skipped_lines_still_count(self, tmp_path): + """A row is named by the line a reader would look at.""" + path = _write(tmp_path, "# dims: time\na,b\n1,2,3\n") + with pytest.raises(ParameterError, match="row 3"): + read_table(path, skip=1) + class TestRowCells: """Only stated cells are reported.""" @@ -197,3 +223,197 @@ def test_float_keeps_its_point(self): def test_text(self): """Anything else is the string it was written as.""" assert parse_cell("car") == "car" + + +def _forge(frame: pd.DataFrame, path, documents: str) -> None: + """Write a parquet file whose document footer this library did not write.""" + table = pyarrow.Table.from_pandas(frame, preserve_index=False) + kept = {**(table.schema.metadata or {}), DOCUMENT_KEY: documents} + pyarrow.parquet.write_table(table.replace_schema_metadata(kept), path) + + +@pytest.mark.skipif(pyarrow is None, reason="pyarrow is not installed") +class TestParquet: + """Parquet keeps what a column holds, and what the file says about itself.""" + + def test_types_survive(self, tmp_path): + """A column comes back as what it was written as.""" + frame = pd.DataFrame({"a": [1.5], "b": [True], "c": ["text"]}) + path = tmp_path / "table.parquet" + write_parquet(frame, path) + out, _ = read_parquet(path) + assert out.equals(frame) + + def test_a_column_of_no_one_type(self, tmp_path): + """A column parquet has no shape for is written as documents.""" + frame = pd.DataFrame({"value": ["car", True, 3]}) + path = tmp_path / "table.parquet" + write_parquet(frame, path) + out, _ = read_parquet(path) + assert list(out["value"]) == ["car", True, 3] + + def test_a_nested_cell(self, tmp_path): + """A mapping is a document, and reads back as the mapping it was.""" + path = tmp_path / "table.parquet" + write_parquet(pd.DataFrame({"meta": [{"a": [1, 2]}, None]}), path) + out, _ = read_parquet(path) + assert list(out["meta"]) == [{"a": [1, 2]}, None] + + def test_a_column_of_text_stays_a_column_of_text(self, tmp_path): + """Text is a type parquet has, so it is not written as documents.""" + frame = pd.DataFrame({"note": pd.Series(["a", None], dtype=object)}) + assert frame["note"].dtype == object # the branch this is about + table = parquet_table(frame) + assert DOCUMENT_KEY.encode() not in (table.schema.metadata or {}) + # Whichever width of string arrow picks, it is a string column. + assert "string" in str(table.schema.field("note").type) + path = tmp_path / "table.parquet" + write_parquet(frame, path) + out, _ = read_parquet(path) + assert out["note"][0] == "a" and pd.isna(out["note"][1]) + + def test_a_cell_holding_a_list(self, tmp_path): + """A container states itself; asking pandas answers once per element.""" + path = tmp_path / "table.parquet" + write_parquet(pd.DataFrame({"tags": [["a", "b"], None]}), path) + out, _ = read_parquet(path) + assert list(out["tags"]) == [["a", "b"], None] + + def test_numbers_numpy_made(self, tmp_path): + """A numpy scalar is the number it holds, not the text str() gives.""" + path = tmp_path / "table.parquet" + frame = pd.DataFrame({"value": [np.int64(3), "text", np.bool_(True)]}) + write_parquet(frame, path) + out, _ = read_parquet(path) + assert list(out["value"]) == [3, "text", True] + + def test_a_time_of_any_spelling(self, tmp_path): + """A time has no JSON type, so every spelling of one is written alike.""" + path = tmp_path / "table.parquet" + stamp = pd.Timestamp("2020-01-01T00:00:01") + # Three spellings of one instant, which arrive at different + # resolutions: a Timestamp reads back at microseconds, the others at + # nanoseconds, and the file holds one of them. + frame = pd.DataFrame( + {"when": [stamp, stamp.to_pydatetime(), stamp.to_datetime64(), "text"]} + ) + write_parquet(frame, path) + out, _ = read_parquet(path) + written = list(out["when"]) + assert written[3] == "text" + assert len(set(written[:3])) == 1 + assert to_datetime64(written[0]) == np.datetime64("2020-01-01T00:00:01") + + def test_a_duration_and_a_model(self, tmp_path): + """Neither has a JSON type; a model dumps itself, a duration is text.""" + path = tmp_path / "table.parquet" + line = Line(start={"distance": 0.0}, end={"distance": 10.0}) + frame = pd.DataFrame({"cell": [np.timedelta64(5, "s"), line, "text"]}) + write_parquet(frame, path) + out, _ = read_parquet(path) + held = list(out["cell"]) + assert to_timedelta64(held[0]) == np.timedelta64(5, "s") + assert Line(**held[1]) == line + + def test_a_duration_of_any_spelling(self, tmp_path): + """A duration is written alike however it arrived, as a time is.""" + path = tmp_path / "table.parquet" + span = pd.Timedelta(seconds=5) + frame = pd.DataFrame( + {"span": [span, span.to_pytimedelta(), span.to_numpy(), "text"]} + ) + write_parquet(frame, path) + out, _ = read_parquet(path) + written = list(out["span"]) + assert written[3] == "text" + assert len(set(written[:3])) == 1 + assert to_timedelta64(written[0]) == np.timedelta64(5, "s") + + def test_a_missing_value_inside_a_document(self, tmp_path): + """Missing is missing, not the text of whichever spelling it arrived in.""" + path = tmp_path / "table.parquet" + write_parquet(pd.DataFrame({"meta": [{"score": pd.NA, "n": 1}, "text"]}), path) + out, _ = read_parquet(path) + assert list(out["meta"]) == [{"score": None, "n": 1}, "text"] + + def test_a_document_column_keeps_its_own_type(self, tmp_path): + """A column of documents holds what its cells hold, not what pandas infers.""" + path = tmp_path / "table.parquet" + write_parquet(pd.DataFrame({"code": pd.Series([1, 2], dtype=object)}), path) + out, _ = read_parquet(path) + assert out["code"].dtype == object + assert list(out["code"]) == [1, 2] + + def test_metadata_round_trips(self, tmp_path): + """What a format states about its table comes back to it.""" + path = tmp_path / "table.parquet" + write_parquet(pd.DataFrame({"a": [1]}), path, {"dascore:dims": '["time"]'}) + _, stated = read_parquet(path) + assert stated == {"dascore:dims": '["time"]'} + + def test_metadata_without_the_table(self, tmp_path): + """The footer alone, for a caller which only needs what it says.""" + path = tmp_path / "table.parquet" + write_parquet(pd.DataFrame({"a": [1]}), path, {"dascore:dims": '["time"]'}) + assert read_parquet_metadata(path) == {"dascore:dims": '["time"]'} + + def test_no_columns_refused(self, tmp_path): + """An empty table states nothing, as an empty CSV does.""" + path = tmp_path / "table.parquet" + write_parquet(pd.DataFrame(), path) + with pytest.raises(ParameterError, match="states no track"): + read_parquet(path, what="no track") + + def test_a_file_which_is_not_parquet(self, tmp_path): + """Whatever pyarrow raises, the caller sees its own error.""" + path = _write(tmp_path, "a,b\n1,2\n", name="table.parquet") + with pytest.raises(ParameterError, match="Could not read"): + read_parquet(path) + with pytest.raises(ParameterError, match="Could not read"): + read_parquet_metadata(path) + + @pytest.mark.parametrize("key", [DOCUMENT_KEY, "pandas", "ARROW:schema"]) + def test_a_key_the_file_states_for_itself(self, key, tmp_path): + """A caller states neither this writer's keys nor the format's own.""" + path = tmp_path / "table.parquet" + with pytest.raises(ParameterError, match="not a caller's to state"): + write_parquet(pd.DataFrame({"a": [1]}), path, {key: '["a"]'}) + + def test_a_column_named_something_other_than_text(self, tmp_path): + """Arrow names a column with text, so a label is spelled as text.""" + path = tmp_path / "table.parquet" + write_parquet(pd.DataFrame({"distance": [1.0], 7: ["x"]}), path) + out, _ = read_parquet(path) + assert list(out.columns) == ["distance", "7"] + + def test_two_labels_which_spell_alike(self, tmp_path): + """One column states one thing, whatever the two labels were.""" + path = tmp_path / "table.parquet" + frame = pd.DataFrame({7: ["a"], "7": ["b"]}) + with pytest.raises(ParameterError, match="named more than once"): + write_parquet(frame, path) + + def test_a_file_whose_pandas_metadata_is_not_json(self, tmp_path): + """Another writer's metadata is read here, so a bad one is named here.""" + path = tmp_path / "table.parquet" + table = pyarrow.Table.from_pandas( + pd.DataFrame({"a": [1]}), preserve_index=False + ) + kept = {**(table.schema.metadata or {}), b"pandas": b"not json"} + pyarrow.parquet.write_table(table.replace_schema_metadata(kept), path) + with pytest.raises(ParameterError, match="Could not read"): + read_parquet(path) + + def test_a_document_column_which_is_not_there(self, tmp_path): + """A file naming a column it does not hold says so.""" + path = tmp_path / "table.parquet" + _forge(pd.DataFrame({"a": [1]}), path, '["b"]') + with pytest.raises(ParameterError, match="holds no such column"): + read_parquet(path) + + def test_a_document_which_does_not_parse(self, tmp_path): + """A cell a file names as a document is read as one, or named.""" + path = tmp_path / "table.parquet" + _forge(pd.DataFrame({"a": ["{oops"]}), path, '["a"]') + with pytest.raises(ParameterError, match="not a JSON document"): + read_parquet(path) diff --git a/tests/test_utils/test_time.py b/tests/test_utils/test_time.py index 0f6c6d145..8abeec9a9 100644 --- a/tests/test_utils/test_time.py +++ b/tests/test_utils/test_time.py @@ -13,6 +13,11 @@ import dascore as dc from dascore.compat import random_state from dascore.exceptions import TimeError, UnitError + +try: + import pyarrow +except ImportError: + pyarrow = None from dascore.utils.time import ( is_datetime64, is_timedelta64, @@ -238,6 +243,16 @@ def test_pandas_string_array(self): ) assert np.all(out == expected) + @pytest.mark.skipif(pyarrow is None, reason="pyarrow is not installed") + def test_arrow_backed_string_array(self): + """Which backing pandas gives text is not the caller's choice.""" + arr = pd.array(self.date_strs, dtype="string[pyarrow]") + out = to_datetime64(arr) + expected = np.array([dc.to_datetime64(x) for x in self.date_strs]).astype( + "datetime64[ns]" + ) + assert np.all(out == expected) + class TestToTimeDelta64: """Tests for creating timedeltas.""" @@ -364,6 +379,13 @@ def test_pandas_string_array(self): assert np.all(out[:2] == expected[:2]) assert pd.isnull(out[2]) + @pytest.mark.skipif(pyarrow is None, reason="pyarrow is not installed") + def test_arrow_backed_string_array(self): + """A string column is arrow-backed wherever pyarrow is installed.""" + arr = pd.array(["1s", "2s"], dtype="string[pyarrow]") + out = to_timedelta64(arr) + assert np.all(out == np.array([1, 2]).astype("timedelta64[s]")) + def test_unsupported_type(self): """Ensure unsupported types raise.""" with pytest.raises(NotImplementedError):