diff --git a/dascore/core/annotation_loader.py b/dascore/core/annotation_loader.py index 59364a10e..be2bfd439 100644 --- a/dascore/core/annotation_loader.py +++ b/dascore/core/annotation_loader.py @@ -19,6 +19,10 @@ carries the annotations made on it under the hidden name ``.annotations``, as it carries its inventory under ``.inventory``. +A column whose header begins with an underscore is the author's own -- a +crew's notes on how something was deployed, say -- and is read by nothing: +the set does not carry it, so it stays in the file it was written in. + CSV has no types, so this module decides what each column holds before the models see it: a ``basis`` cell is the JSON document its curve dumps, and every other cell is read the way it was written. A dimension column is @@ -66,6 +70,7 @@ from dascore.utils.misc import iterate from dascore.utils.paths import quote_path from dascore.utils.tables import ( + drop_private_columns, parse_cell, read_parquet, read_parquet_metadata, @@ -261,7 +266,10 @@ def _read_cells( out[name] = series else: out[name] = series.map(_read_extra) - return pd.DataFrame(out) + # The index the table was read with: a file whose every column is the + # author's own still stated rows, and building from the columns alone + # would drop them where nothing would say they had gone. + return pd.DataFrame(out, index=frame.index) def _check_kind(series: pd.Series, name, path: Path, kinds: str, what: str) -> None: @@ -306,13 +314,34 @@ def _read_set_table( frame, _ = read_parquet(path, what=what, empty=True) if not len(frame.columns): return None + frame = _kept_columns(frame, path) return _read_cells(frame, dims, path, ordered=ordered, typed=True, text=text) if _is_blank(path): return None - frame = read_table(path, what=what, skip=skip) + frame = _kept_columns(read_table(path, what=what, skip=skip), path) return _read_cells(frame, dims, path, ordered=ordered, text=text) +def _kept_columns(frame: pd.DataFrame, path: Path) -> pd.DataFrame: + """ + Return a table without the columns its author kept for themselves. + + Read before any cell is: what a private column holds is not this + format's to type, to check against a declaration, or to refuse. A + table of nothing else states rows no column of the set can hold, and + is named here rather than left to the set, which would no longer know + which file they were in. + """ + kept = drop_private_columns(frame) + if len(kept.index) and not len(kept.columns): + msg = ( + f"{quote_path(path)} states rows and no column but its author's " + "own; a header beginning with an underscore is read by nothing." + ) + raise ParameterError(msg) + return kept + + def _is_parquet(path: Path) -> bool: """Whether a table's name says it is parquet rather than CSV.""" return path.suffix.casefold() == PARQUET_SUFFIX diff --git a/dascore/core/annotations.py b/dascore/core/annotations.py index 2217a8222..b26116d3a 100644 --- a/dascore/core/annotations.py +++ b/dascore/core/annotations.py @@ -14,6 +14,11 @@ and polygons keep their vertices in a second, tidy frame keyed by annotation id, and every row keeps a bounding region so table operations work whatever its geometry is. + +Any other column is an extra the annotation carries, with one exception: a +column whose name begins with an underscore is the author's own record +keeping. A set never holds one, so it stays where it was written and no +reader looks for meaning in it. """ from __future__ import annotations @@ -68,6 +73,8 @@ from dascore.utils.misc import iterate, to_str, validate_acquisition_key from dascore.utils.namespace import NamespaceOwner from dascore.utils.tables import ( + PRIVATE_PREFIX, + drop_private_columns, parquet_table, parse_cell, write_parquet, @@ -695,6 +702,15 @@ def _check_dims(self) -> Self: f"a set may not dimension {', '.join(RESERVED_COLUMNS)}." ) raise ValueError(msg) + # A dimension is stated by a column, and a private column is the + # author's own: dimensioning one would declare a coordinate no + # table is allowed to hold. + if private := sorted(x for x in self.dims if x.startswith(PRIVATE_PREFIX)): + msg = ( + f"The dimension(s) {', '.join(private)} begin with an " + "underscore, which names a column no set reads." + ) + raise ValueError(msg) return self @model_validator(mode="after") @@ -1036,7 +1052,20 @@ def _coerce_frame(data, what: str) -> pd.DataFrame: "than a string, which is what a table names a column by." ) raise ParameterError(msg) - return frame + # Here rather than where a file is read, so a set holds what a stored + # one holds: a private column is the author's own either way, and a set + # which kept one from a frame would write a column it could not read + # back. + kept = drop_private_columns(frame) + # A table writes rows by writing their cells, so rows with no cell to + # write are rows a saved set comes back without. Refused rather than + # counted here, where what went missing can still be named. + if len(kept.index) and not len(kept.columns): + msg = f"The {what} state rows and no column to hold them." + if len(frame.columns): + msg += " Every column they state is private, so none is theirs." + raise ParameterError(msg) + return kept def _read_spellings(frame: pd.DataFrame, dims) -> dict[str, _Spelling]: diff --git a/dascore/core/inventory_loader.py b/dascore/core/inventory_loader.py index e075fe72a..e6d9dacfd 100644 --- a/dascore/core/inventory_loader.py +++ b/dascore/core/inventory_loader.py @@ -25,7 +25,10 @@ Loading is strict about near-misses and indifferent to clean misses: anything which claims to participate in a convention and gets it wrong raises, while anything which does not participate -- photos, field notes, -deployment logs -- is ignored where it lies. +deployment logs -- is ignored where it lies. A column does the same: a +header beginning with an underscore is the crew's own record keeping, and +no table reads it. What it holds stays in the file, so a note which should +travel with the inventory goes in ``description`` instead. """ from __future__ import annotations @@ -64,6 +67,7 @@ from dascore.utils.misc import check_code from dascore.utils.paths import quote_path as _quote from dascore.utils.tables import ( + drop_private_columns, ordered_rows, parse_cell, read_table, @@ -885,6 +889,11 @@ def _read_track_table(path: Path, table: _Table, stem: str, crs): if frame.empty: msg = f"{_quote(path)} states no rows, so it describes no {stem}." raise InvalidInventoryError(msg) + # Read before any header is: a private column is the author's own, so + # a geometry table's numeric rule and the model's unknown-field error + # are both none of its business. A table of nothing else keeps its + # rows, and is refused by the columns it then fails to state. + frame = drop_private_columns(frame) units: Mapping[str, str] = {} if stem == "geometry": frame, units = _geometry_columns(frame, crs, path) diff --git a/dascore/utils/tables.py b/dascore/utils/tables.py index 5237b2c37..797ad0902 100644 --- a/dascore/utils/tables.py +++ b/dascore/utils/tables.py @@ -32,6 +32,40 @@ # The metadata key a parquet file names its document columns in. DOCUMENT_KEY = "dascore:documents" +# What a column names itself by to state that it is the author's own. +PRIVATE_PREFIX = "_" + + +def drop_private_columns(frame: pd.DataFrame) -> pd.DataFrame: + """ + Return a table without the columns which say they are not its own. + + A header beginning with an underscore declines to take part in the + format reading it: the column is the crew's own record keeping -- who + backfilled a trench, which drawing a run came from -- and no reader + looks for meaning in it. The name can never collide with a field a + model might later add, since pydantic makes a leading underscore a + private attribute rather than a field. + + The values live in the file alone. A note which should travel with the + data belongs in a field the model has, `description` among them. + + Parameters + ---------- + frame + The table to read the private columns out of. + + Examples + -------- + >>> import pandas as pd + >>> from dascore.utils.tables import drop_private_columns + >>> frame = pd.DataFrame({"group": ["rail"], "_crew": ["mapped by JD"]}) + >>> list(drop_private_columns(frame).columns) + ['group'] + """ + private = [x for x in frame.columns if str(x).startswith(PRIVATE_PREFIX)] + return frame.drop(columns=private) if private else frame + def read_table(path: Path, what: str = "nothing", skip: int = 0) -> pd.DataFrame: r""" @@ -275,6 +309,11 @@ def read_parquet( "and holds no such column." ) raise ParameterError(msg) + # A private column is the author's own, so what its cells hold is + # not read here either: a document in one is left as the text it + # is, as the same column in a CSV is. + if str(name).startswith(PRIVATE_PREFIX): + continue # Held as object: the cells are whatever their documents state, and # letting pandas re-infer a type from them would hand back a column # of a type the file never said it had. diff --git a/docs/tutorial/inventory.qmd b/docs/tutorial/inventory.qmd index 533f67313..b8deed4de 100644 --- a/docs/tutorial/inventory.qmd +++ b/docs/tutorial/inventory.qmd @@ -135,6 +135,7 @@ A few rules make that directory readable without a schema in front of you: - **The name states the identity.** `acquisitions/DAS.R2D1..RAW.yaml` *is* the acquisition `DAS.R2D1..RAW` — network `DAS`, fiber array `R2D1`, blank location code, code `RAW`. Restating a part of it inside the file is allowed, as long as the two agree; there is never a precedence rule between two spellings of one fact. - **`path` is the one reserved *container* name.** A directory called `path` addresses an optical path rather than serializing an attribute named "path". (`attrs` and `inventory` are reserved as file stems — an entity's own object file, and the envelope — and the top-level directory names above are fixed.) - **Files that participate in no convention are ignored.** Photos, field notes, and deployment logs can live in the inventory directory undisturbed. +- **Columns that participate in no convention are ignored too.** A CSV header beginning with an underscore — `_crew`, `_drawing` — is the crew's own record keeping, and nothing reads it. It stays in the file it was written in, so a note which should travel with the inventory goes in a `description` column instead, which every object has. Two tracks can hold something that varies along the fiber, and which one to use is decided by how a value behaves between the points which state it. **A quantity which interpolates goes in `geometry.csv`**, one column per quantity, read as a curve through its control points; a column may name its units in its header, `chainage (m)`. **A value which holds over a stretch and then stops goes in `labels.csv`**, a set of intervals rather than a curve — a zone name has no meaning between two of them, and neither does a borehole number. Labels are usually words, then, but a number which identifies rather than measures is a label too. A column of text in a geometry table is refused, and says so. diff --git a/tests/test_core/test_annotation_loader.py b/tests/test_core/test_annotation_loader.py index 703180e60..dea85f1e1 100644 --- a/tests/test_core/test_annotation_loader.py +++ b/tests/test_core/test_annotation_loader.py @@ -1685,6 +1685,73 @@ def _forge(frame: pd.DataFrame, path, documents: str) -> None: pyarrow.parquet.write_table(table.replace_schema_metadata(kept), path) +class TestPrivateColumns: + """A column an author kept for themselves is read by nothing.""" + + def test_a_bare_table(self, tmp_path): + """A note on how something was deployed stays in the file.""" + path = tmp_path / "picks.csv" + path.write_text( + "id,group,distance_start,distance_end,_crew\n" + "r1,noise,10.0,60.0,north crew\n" + ) + loaded = dc.annotations(path, dims=DIMS) + assert "_crew" not in loaded.io.to_dataframe().columns + assert loaded[0].group == "noise" + + def test_a_saved_set(self, regions, tmp_path): + """One added to a written table changes nothing about the set.""" + directory = regions.io.save(tmp_path / "picks") + table = directory / "annotations.csv" + header, *rows = table.read_text().splitlines() + written = [f"{header},_crew", *[f"{row},north crew" for row in rows]] + table.write_text("\n".join(written) + "\n") + assert dc.annotations(directory) == regions + + def test_nothing_reads_what_it_holds(self, tmp_path): + """A declaration a private column cannot meet is not checked.""" + directory = tmp_path / "picks" + directory.mkdir() + (directory / "annotations.csv").write_text( + "id,distance,_count\nr1,1.0,not a number\n" + ) + (directory / "attrs.yaml").write_text( + yaml.safe_dump( + { + "object_type": "AnnotationSetAttrs", + "dims": list(DIMS), + "columns": {"_count": {"dtype": "Int64"}}, + } + ) + ) + assert len(dc.annotations(directory)) == 1 + + @pytest.mark.skipif(pyarrow is None, reason="pyarrow is not installed") + def test_a_private_document_column(self, tmp_path): + """Parquet reads one no further than a CSV does.""" + frame = pd.DataFrame({"id": ["r1"], "distance": [1.0], "_crew": ["{oops"]}) + path = tmp_path / "picks.parquet" + _forge(frame, path, '["_crew"]') + loaded = dc.annotations(path, dims=DIMS) + assert "_crew" not in loaded.io.to_dataframe().columns + + def test_a_table_of_only_private_columns(self, tmp_path): + """Rows no column of the set states are refused, not lost.""" + path = tmp_path / "picks.csv" + path.write_text("_crew\nnorth crew\n") + with pytest.raises(InvalidAnnotationError, match="read by nothing"): + dc.annotations(path, dims=DIMS) + + def test_vertices(self, with_vertices, tmp_path): + """Vertices are a table like any other, so they take one too.""" + directory = with_vertices.io.save(tmp_path / "picks") + table = directory / "vertices.csv" + header, *rows = table.read_text().splitlines() + written = [f"{header},_source", *[f"{row},drawing 4" for row in rows]] + table.write_text("\n".join(written) + "\n") + assert dc.annotations(directory) == with_vertices + + @pytest.mark.skipif(pyarrow is None, reason="pyarrow is not installed") class TestParquet: """The same tables, with their types kept, for a set too big to want text.""" diff --git a/tests/test_core/test_annotations.py b/tests/test_core/test_annotations.py index 8ef02a220..e82511b74 100644 --- a/tests/test_core/test_annotations.py +++ b/tests/test_core/test_annotations.py @@ -419,6 +419,38 @@ def test_no_set_column_is_no_label(self): out = AnnotationSet(pd.DataFrame({"group": ["a"]}), dims=DIMS) assert out[0].set == "" + def test_private_column_is_not_an_extra(self): + """An underscore says the column is the author's, not the set's.""" + frame = pd.DataFrame({"score": [0.9], "_crew": ["north crew"]}) + out = AnnotationSet(frame, dims=DIMS) + assert "_crew" not in out[0].extra + assert "_crew" not in out.io.to_dataframe().columns + + def test_a_private_column_states_no_dimension(self): + """Underscoring a range column makes it nothing, not a bound.""" + frame = pd.DataFrame( + {"group": ["a"], "_distance_start": [1.0], "_distance_end": [2.0]} + ) + out = AnnotationSet(frame, dims=DIMS) + assert out[0].region.bounds == {} + + def test_a_private_dimension(self): + """A dimension is a column, and no set reads a private one.""" + frame = pd.DataFrame({"_distance": [1.0]}) + with pytest.raises(ValidationError, match="begin with an underscore"): + AnnotationSet(frame, dims=("_distance", "time")) + + def test_rows_no_column_can_hold(self): + """Rows a table cannot write are refused where they can be named.""" + frame = pd.DataFrame({"_crew": ["north crew", "south crew"]}) + with pytest.raises(ParameterError, match="none is theirs"): + AnnotationSet(frame, dims=DIMS) + + def test_rows_which_state_nothing_at_all(self): + """The same, for a frame which never had a column to lose.""" + with pytest.raises(ParameterError, match="no column to hold them"): + AnnotationSet(pd.DataFrame(index=range(2)), dims=DIMS) + def test_declared_column_documents_only(self): """Documenting a column does not gate any other one.""" out = AnnotationSet( diff --git a/tests/test_core/test_inventory_loader.py b/tests/test_core/test_inventory_loader.py index 697dfdf6e..c943e98ef 100644 --- a/tests/test_core/test_inventory_loader.py +++ b/tests/test_core/test_inventory_loader.py @@ -1498,6 +1498,67 @@ def test_a_column_stated_twice(self, make_inventory): make_inventory(files) +class TestPrivateColumns: + """A header beginning with an underscore is the crew's own.""" + + @staticmethod + def _annotated(files, column="_crew_note"): + """Return the track tables with one private column added to each.""" + out = dict(files) + for name, text in files.items(): + if not name.endswith(".csv"): + continue + header, *rows = text.splitlines() + written = [f"{header},{column}"] + written += [f"{row},mapped from drawing 4" for row in rows] + out[name] = "\n".join(written) + "\n" + return out + + def test_every_table_takes_one(self, make_inventory): + """The path a private column is added to is the path without it.""" + plain = one_path(make_inventory({**MINIMAL, **TRACKS}, name="plain")) + noted = one_path( + make_inventory({**MINIMAL, **self._annotated(TRACKS)}, name="noted") + ) + assert noted == plain + + def test_geometry_takes_text(self, make_inventory): + """A geometry column is numeric; a private one is not a column.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/geometry.csv": ( + "segment,distance,longitude,latitude,elevation,_surveyed_by\n" + "S100,100.0,-117.0,40.0,687.0,north crew\n" + "S100,102.0,-117.1,40.1,685.0,north crew\n" + ), + } + segment = one_path(make_inventory(files)).geometry[0] + assert set(segment.coordinates) == {"longitude", "latitude", "elevation"} + + def test_a_table_of_only_private_columns(self, make_inventory): + """A table which states nothing of its own states no track.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/coupling.csv": "_crew\nnorth crew\n", + } + with pytest.raises(InvalidInventoryError, match="start_distance"): + make_inventory(files) + + def test_a_private_column_states_nothing(self, make_inventory): + """Underscoring a column the table needs does not state it.""" + files = { + **MINIMAL, + **TRACKS, + "fiber_arrays/DAS.L001/path/coupling.csv": ( + "_start_distance,end_distance,coupling_type\n0,340,conduit\n" + ), + } + with pytest.raises(InvalidInventoryError, match="start_distance"): + make_inventory(files) + + class TestPathEpochs: """`path` is the one reserved container stem, and it holds a lineage.""" diff --git a/tests/test_utils/test_tables.py b/tests/test_utils/test_tables.py index 99eefd7cb..44a56f83f 100644 --- a/tests/test_utils/test_tables.py +++ b/tests/test_utils/test_tables.py @@ -12,6 +12,7 @@ from dascore.exceptions import ParameterError from dascore.utils.tables import ( DOCUMENT_KEY, + drop_private_columns, ordered_rows, parquet_table, parse_cell, @@ -116,6 +117,31 @@ def test_skipped_lines_still_count(self, tmp_path): read_table(path, skip=1) +class TestDropPrivateColumns: + """A column naming itself private is nobody's to read.""" + + def test_private_columns_go(self): + """An underscore says the column is not the format's.""" + frame = pd.DataFrame({"group": ["rail"], "_crew": ["JD"], "_": ["x"]}) + assert list(drop_private_columns(frame).columns) == ["group"] + + def test_a_table_without_one_is_the_table(self): + """Nothing to drop leaves the frame as it was.""" + frame = pd.DataFrame({"group": ["rail"]}) + assert drop_private_columns(frame) is frame + + def test_the_frame_given_is_not_changed(self): + """Dropping hands back a new table, as the rest of this module does.""" + frame = pd.DataFrame({"group": ["rail"], "_crew": ["JD"]}) + drop_private_columns(frame) + assert "_crew" in frame.columns + + def test_an_underscore_inside_a_name_stays(self): + """The prefix states the intent; a name merely holding one does not.""" + frame = pd.DataFrame({"start_distance": [0.0], "mid_": [1.0]}) + assert list(drop_private_columns(frame).columns) == ["start_distance", "mid_"] + + class TestRowCells: """Only stated cells are reported.""" @@ -259,6 +285,14 @@ def test_types_survive(self, tmp_path): out, _ = read_parquet(path) assert out.equals(frame) + def test_a_private_document_column_is_not_read(self, tmp_path): + """What a private column holds is not this reader's to parse.""" + frame = pd.DataFrame({"group": ["rail"], "_crew": ["{oops"]}) + path = tmp_path / "table.parquet" + _forge(frame, path, '["_crew"]') + out, _ = read_parquet(path) + assert out["_crew"][0] == "{oops" + def test_a_column_of_no_one_type(self, tmp_path): """A column parquet has no shape for is written as documents.""" frame = pd.DataFrame({"value": ["car", True, 3]})