Let an author keep a column to themselves

dascore/core/annotation_loader.py

@@ -19,6 +19,10 @@
 carries the annotations made on it under the hidden name ``.annotations``, as
 it carries its inventory under ``.inventory``.
 
+A column whose header begins with an underscore is the author's own -- a
+crew's notes on how something was deployed, say -- and is read by nothing:
+the set does not carry it, so it stays in the file it was written in.
+
 CSV has no types, so this module decides what each column holds before the
 models see it: a ``basis`` cell is the JSON document its curve dumps, and
 every other cell is read the way it was written. A dimension column is
@@ -66,6 +70,7 @@
 from dascore.utils.misc import iterate
 from dascore.utils.paths import quote_path
 from dascore.utils.tables import (
+    drop_private_columns,
     parse_cell,
     read_parquet,
     read_parquet_metadata,
@@ -261,7 +266,10 @@ def _read_cells(
             out[name] = series
         else:
             out[name] = series.map(_read_extra)
-    return pd.DataFrame(out)
+    # The index the table was read with: a file whose every column is the
+    # author's own still stated rows, and building from the columns alone
+    # would drop them where nothing would say they had gone.
+    return pd.DataFrame(out, index=frame.index)
 
 
 def _check_kind(series: pd.Series, name, path: Path, kinds: str, what: str) -> None:
@@ -306,13 +314,34 @@ def _read_set_table(
         frame, _ = read_parquet(path, what=what, empty=True)
         if not len(frame.columns):
             return None
+        frame = _kept_columns(frame, path)
         return _read_cells(frame, dims, path, ordered=ordered, typed=True, text=text)
     if _is_blank(path):
         return None
-    frame = read_table(path, what=what, skip=skip)
+    frame = _kept_columns(read_table(path, what=what, skip=skip), path)
     return _read_cells(frame, dims, path, ordered=ordered, text=text)
 
 
+def _kept_columns(frame: pd.DataFrame, path: Path) -> pd.DataFrame:
+    """
+    Return a table without the columns its author kept for themselves.
+
+    Read before any cell is: what a private column holds is not this
+    format's to type, to check against a declaration, or to refuse. A
+    table of nothing else states rows no column of the set can hold, and
+    is named here rather than left to the set, which would no longer know
+    which file they were in.
+    """
+    kept = drop_private_columns(frame)
+    if len(kept.index) and not len(kept.columns):
+        msg = (
+            f"{quote_path(path)} states rows and no column but its author's "
+            "own; a header beginning with an underscore is read by nothing."
+        )
+        raise ParameterError(msg)
+    return kept
+
+
 def _is_parquet(path: Path) -> bool:
     """Whether a table's name says it is parquet rather than CSV."""
     return path.suffix.casefold() == PARQUET_SUFFIX

dascore/core/annotations.py

@@ -14,6 +14,11 @@
 and polygons keep their vertices in a second, tidy frame keyed by
 annotation id, and every row keeps a bounding region so table operations
 work whatever its geometry is.
+
+Any other column is an extra the annotation carries, with one exception: a
+column whose name begins with an underscore is the author's own record
+keeping. A set never holds one, so it stays where it was written and no
+reader looks for meaning in it.
 """
 
 from __future__ import annotations
@@ -68,6 +73,8 @@
 from dascore.utils.misc import iterate, to_str, validate_acquisition_key
 from dascore.utils.namespace import NamespaceOwner
 from dascore.utils.tables import (
+    PRIVATE_PREFIX,
+    drop_private_columns,
     parquet_table,
     parse_cell,
     write_parquet,
@@ -695,6 +702,15 @@ def _check_dims(self) -> Self:
                 f"a set may not dimension {', '.join(RESERVED_COLUMNS)}."
             )
             raise ValueError(msg)
+        # A dimension is stated by a column, and a private column is the
+        # author's own: dimensioning one would declare a coordinate no
+        # table is allowed to hold.
+        if private := sorted(x for x in self.dims if x.startswith(PRIVATE_PREFIX)):
+            msg = (
+                f"The dimension(s) {', '.join(private)} begin with an "
+                "underscore, which names a column no set reads."
+            )
+            raise ValueError(msg)
         return self
 
     @model_validator(mode="after")
@@ -1036,7 +1052,20 @@ def _coerce_frame(data, what: str) -> pd.DataFrame:
             "than a string, which is what a table names a column by."
         )
         raise ParameterError(msg)
-    return frame
+    # Here rather than where a file is read, so a set holds what a stored
+    # one holds: a private column is the author's own either way, and a set
+    # which kept one from a frame would write a column it could not read
+    # back.
+    kept = drop_private_columns(frame)
+    # A table writes rows by writing their cells, so rows with no cell to
+    # write are rows a saved set comes back without. Refused rather than
+    # counted here, where what went missing can still be named.
+    if len(kept.index) and not len(kept.columns):
+        msg = f"The {what} state rows and no column to hold them."
+        if len(frame.columns):
+            msg += " Every column they state is private, so none is theirs."
+        raise ParameterError(msg)
+    return kept
 
 
 def _read_spellings(frame: pd.DataFrame, dims) -> dict[str, _Spelling]:

dascore/core/inventory_loader.py

@@ -25,7 +25,10 @@
 Loading is strict about near-misses and indifferent to clean misses:
 anything which claims to participate in a convention and gets it wrong
 raises, while anything which does not participate -- photos, field notes,
-deployment logs -- is ignored where it lies.
+deployment logs -- is ignored where it lies. A column does the same: a
+header beginning with an underscore is the crew's own record keeping, and
+no table reads it. What it holds stays in the file, so a note which should
+travel with the inventory goes in ``description`` instead.
 """
 
 from __future__ import annotations
@@ -64,6 +67,7 @@
 from dascore.utils.misc import check_code
 from dascore.utils.paths import quote_path as _quote
 from dascore.utils.tables import (
+    drop_private_columns,
     ordered_rows,
     parse_cell,
     read_table,
@@ -885,6 +889,11 @@ def _read_track_table(path: Path, table: _Table, stem: str, crs):
     if frame.empty:
         msg = f"{_quote(path)} states no rows, so it describes no {stem}."
         raise InvalidInventoryError(msg)
+    # Read before any header is: a private column is the author's own, so
+    # a geometry table's numeric rule and the model's unknown-field error
+    # are both none of its business. A table of nothing else keeps its
+    # rows, and is refused by the columns it then fails to state.
+    frame = drop_private_columns(frame)
     units: Mapping[str, str] = {}
     if stem == "geometry":
         frame, units = _geometry_columns(frame, crs, path)

dascore/utils/tables.py

@@ -32,6 +32,40 @@
 # The metadata key a parquet file names its document columns in.
 DOCUMENT_KEY = "dascore:documents"
 
+# What a column names itself by to state that it is the author's own.
+PRIVATE_PREFIX = "_"
+
+
+def drop_private_columns(frame: pd.DataFrame) -> pd.DataFrame:
+    """
+    Return a table without the columns which say they are not its own.
+
+    A header beginning with an underscore declines to take part in the
+    format reading it: the column is the crew's own record keeping -- who
+    backfilled a trench, which drawing a run came from -- and no reader
+    looks for meaning in it. The name can never collide with a field a
+    model might later add, since pydantic makes a leading underscore a
+    private attribute rather than a field.
+
+    The values live in the file alone. A note which should travel with the
+    data belongs in a field the model has, `description` among them.
+
+    Parameters
+    ----------
+    frame
+        The table to read the private columns out of.
+
+    Examples
+    --------
+    >>> import pandas as pd
+    >>> from dascore.utils.tables import drop_private_columns
+    >>> frame = pd.DataFrame({"group": ["rail"], "_crew": ["mapped by JD"]})
+    >>> list(drop_private_columns(frame).columns)
+    ['group']
+    """
+    private = [x for x in frame.columns if str(x).startswith(PRIVATE_PREFIX)]
+    return frame.drop(columns=private) if private else frame
+
 
 def read_table(path: Path, what: str = "nothing", skip: int = 0) -> pd.DataFrame:
     r"""

docs/tutorial/inventory.qmd

@@ -135,6 +135,7 @@ A few rules make that directory readable without a schema in front of you:
 - **The name states the identity.** `acquisitions/DAS.R2D1..RAW.yaml` *is* the acquisition `DAS.R2D1..RAW` — network `DAS`, fiber array `R2D1`, blank location code, code `RAW`. Restating a part of it inside the file is allowed, as long as the two agree; there is never a precedence rule between two spellings of one fact.
 - **`path` is the one reserved *container* name.** A directory called `path` addresses an optical path rather than serializing an attribute named "path". (`attrs` and `inventory` are reserved as file stems — an entity's own object file, and the envelope — and the top-level directory names above are fixed.)
 - **Files that participate in no convention are ignored.** Photos, field notes, and deployment logs can live in the inventory directory undisturbed.
+- **Columns that participate in no convention are ignored too.** A CSV header beginning with an underscore — `_crew`, `_drawing` — is the crew's own record keeping, and nothing reads it. It stays in the file it was written in, so a note which should travel with the inventory goes in a `description` column instead, which every object has.
 
 Two tracks can hold something that varies along the fiber, and which one to use is decided by how a value behaves between the points which state it. **A quantity which interpolates goes in `geometry.csv`**, one column per quantity, read as a curve through its control points; a column may name its units in its header, `chainage (m)`. **A value which holds over a stretch and then stops goes in `labels.csv`**, a set of intervals rather than a curve — a zone name has no meaning between two of them, and neither does a borehole number. Labels are usually words, then, but a number which identifies rather than measures is a label too. A column of text in a geometry table is refused, and says so.
 

tests/test_core/test_annotation_loader.py

@@ -1686,6 +1686,64 @@ def _forge(frame: pd.DataFrame, path, documents: str) -> None:
 
 
 @pytest.mark.skipif(pyarrow is None, reason="pyarrow is not installed")
+class TestPrivateColumns:
+    """A column an author kept for themselves is read by nothing."""
+
+    def test_a_bare_table(self, tmp_path):
+        """A note on how something was deployed stays in the file."""
+        path = tmp_path / "picks.csv"
+        path.write_text(
+            "id,group,distance_start,distance_end,_crew\n"
+            "r1,noise,10.0,60.0,north crew\n"
+        )
+        loaded = dc.annotations(path, dims=DIMS)
+        assert "_crew" not in loaded.io.to_dataframe().columns
+        assert loaded[0].group == "noise"
+
+    def test_a_saved_set(self, regions, tmp_path):
+        """One added to a written table changes nothing about the set."""
+        directory = regions.io.save(tmp_path / "picks")
+        table = directory / "annotations.csv"
+        header, *rows = table.read_text().splitlines()
+        written = [f"{header},_crew", *[f"{row},north crew" for row in rows]]
+        table.write_text("\n".join(written) + "\n")
+        assert dc.annotations(directory) == regions
+
+    def test_nothing_reads_what_it_holds(self, tmp_path):
+        """A declaration a private column cannot meet is not checked."""
+        directory = tmp_path / "picks"
+        directory.mkdir()
+        (directory / "annotations.csv").write_text(
+            "id,distance,_count\nr1,1.0,not a number\n"
+        )
+        (directory / "attrs.yaml").write_text(
+            yaml.safe_dump(
+                {
+                    "object_type": "AnnotationSetAttrs",
+                    "dims": list(DIMS),
+                    "columns": {"_count": {"dtype": "Int64"}},
+                }
+            )
+        )
+        assert len(dc.annotations(directory)) == 1
+
+    def test_a_table_of_only_private_columns(self, tmp_path):
+        """Rows no column of the set states are refused, not lost."""
+        path = tmp_path / "picks.csv"
+        path.write_text("_crew\nnorth crew\n")
+        with pytest.raises(InvalidAnnotationError, match="read by nothing"):
+            dc.annotations(path, dims=DIMS)
+
+    def test_vertices(self, with_vertices, tmp_path):
+        """Vertices are a table like any other, so they take one too."""
+        directory = with_vertices.io.save(tmp_path / "picks")
+        table = directory / "vertices.csv"
+        header, *rows = table.read_text().splitlines()
+        written = [f"{header},_source", *[f"{row},drawing 4" for row in rows]]
+        table.write_text("\n".join(written) + "\n")
+        assert dc.annotations(directory) == with_vertices
+
+
 class TestParquet:
     """The same tables, with their types kept, for a set too big to want text."""
 

tests/test_core/test_annotations.py

@@ -419,6 +419,38 @@ def test_no_set_column_is_no_label(self):
         out = AnnotationSet(pd.DataFrame({"group": ["a"]}), dims=DIMS)
         assert out[0].set == ""
 
+    def test_private_column_is_not_an_extra(self):
+        """An underscore says the column is the author's, not the set's."""
+        frame = pd.DataFrame({"score": [0.9], "_crew": ["north crew"]})
+        out = AnnotationSet(frame, dims=DIMS)
+        assert "_crew" not in out[0].extra
+        assert "_crew" not in out.io.to_dataframe().columns
+
+    def test_a_private_column_states_no_dimension(self):
+        """Underscoring a range column makes it nothing, not a bound."""
+        frame = pd.DataFrame(
+            {"group": ["a"], "_distance_start": [1.0], "_distance_end": [2.0]}
+        )
+        out = AnnotationSet(frame, dims=DIMS)
+        assert out[0].region.bounds == {}
+
+    def test_a_private_dimension(self):
+        """A dimension is a column, and no set reads a private one."""
+        frame = pd.DataFrame({"_distance": [1.0]})
+        with pytest.raises(ValidationError, match="begin with an underscore"):
+            AnnotationSet(frame, dims=("_distance", "time"))
+
+    def test_rows_no_column_can_hold(self):
+        """Rows a table cannot write are refused where they can be named."""
+        frame = pd.DataFrame({"_crew": ["north crew", "south crew"]})
+        with pytest.raises(ParameterError, match="none is theirs"):
+            AnnotationSet(frame, dims=DIMS)
+
+    def test_rows_which_state_nothing_at_all(self):
+        """The same, for a frame which never had a column to lose."""
+        with pytest.raises(ParameterError, match="no column to hold them"):
+            AnnotationSet(pd.DataFrame(index=range(2)), dims=DIMS)
+
     def test_declared_column_documents_only(self):
         """Documenting a column does not gate any other one."""
         out = AnnotationSet(

tests/test_core/test_inventory_loader.py

@@ -1498,6 +1498,67 @@ def test_a_column_stated_twice(self, make_inventory):
             make_inventory(files)
 
 
+class TestPrivateColumns:
+    """A header beginning with an underscore is the crew's own."""
+
+    @staticmethod
+    def _annotated(files, column="_crew_note"):
+        """Return the track tables with one private column added to each."""
+        out = dict(files)
+        for name, text in files.items():
+            if not name.endswith(".csv"):
+                continue
+            header, *rows = text.splitlines()
+            written = [f"{header},{column}"]
+            written += [f"{row},mapped from drawing 4" for row in rows]
+            out[name] = "\n".join(written) + "\n"
+        return out
+
+    def test_every_table_takes_one(self, make_inventory):
+        """The path a private column is added to is the path without it."""
+        plain = one_path(make_inventory({**MINIMAL, **TRACKS}, name="plain"))
+        noted = one_path(
+            make_inventory({**MINIMAL, **self._annotated(TRACKS)}, name="noted")
+        )
+        assert noted == plain
+
+    def test_geometry_takes_text(self, make_inventory):
+        """A geometry column is numeric; a private one is not a column."""
+        files = {
+            **MINIMAL,
+            **TRACKS,
+            "fiber_arrays/DAS.L001/path/geometry.csv": (
+                "segment,distance,longitude,latitude,elevation,_surveyed_by\n"
+                "S100,100.0,-117.0,40.0,687.0,north crew\n"
+                "S100,102.0,-117.1,40.1,685.0,north crew\n"
+            ),
+        }
+        segment = one_path(make_inventory(files)).geometry[0]
+        assert set(segment.coordinates) == {"longitude", "latitude", "elevation"}
+
+    def test_a_table_of_only_private_columns(self, make_inventory):
+        """A table which states nothing of its own states no track."""
+        files = {
+            **MINIMAL,
+            **TRACKS,
+            "fiber_arrays/DAS.L001/path/coupling.csv": "_crew\nnorth crew\n",
+        }
+        with pytest.raises(InvalidInventoryError, match="start_distance"):
+            make_inventory(files)
+
+    def test_a_private_column_states_nothing(self, make_inventory):
+        """Underscoring a column the table needs does not state it."""
+        files = {
+            **MINIMAL,
+            **TRACKS,
+            "fiber_arrays/DAS.L001/path/coupling.csv": (
+                "_start_distance,end_distance,coupling_type\n0,340,conduit\n"
+            ),
+        }
+        with pytest.raises(InvalidInventoryError, match="start_distance"):
+            make_inventory(files)
+
+
 class TestPathEpochs:
     """`path` is the one reserved container stem, and it holds a lineage."""