ATL03 template + region benchmark + hierarchical-read wiring (Refs #30, #43)

src/zagg/config.py

@@ -455,14 +455,31 @@ def _normalize_filter(f: dict) -> dict:
 
 
 def _validate_filters(data_source: dict) -> None:
-    """Validate the ``filters`` list (and that a flat ``quality_filter`` is sane).
-
-    Raises ``ValueError`` on: unknown op, missing ``dataset``, ``in``/``not_in``
-    without a list ``values``, scalar ops without ``value``, non-int ``column``,
-    a non-base-level ``expression`` filter, or wrong ``value`` type. ``column`` is
-    required for the N-D flag case but cannot be checked against array rank here
-    (no data); rank checks happen at read time.
+    """Validate the ``filters`` list and the flat ``quality_filter`` sugar.
+
+    For the structured ``filters:`` list, raises ``ValueError`` on: unknown op,
+    missing ``dataset``, ``in``/``not_in`` without a list ``values``, scalar ops
+    without ``value``, non-int ``column``, a non-base-level ``expression`` filter,
+    or wrong ``value`` type. ``column`` is required for the N-D flag case but
+    cannot be checked against array rank here (no data); rank checks happen at
+    read time.
+
+    For the flat ``quality_filter`` sugar, only ``dataset`` and ``value`` are
+    honored (``filters_from_data_source`` synthesizes ``op: eq, column: null``).
+    Reject any extra keys at load time so a user-typoed ``op: gt`` or stray
+    ``column:`` is not silently dropped on the floor — the structured ``filters:``
+    list is the right form when those knobs are wanted.
     """
+    qf = data_source.get("quality_filter")
+    if qf is not None:
+        allowed = {"dataset", "value"}
+        unknown = set(qf) - allowed
+        if unknown:
+            raise ValueError(
+                f"data_source.quality_filter only honors {sorted(allowed)} "
+                f"(got extra keys {sorted(unknown)}); use the structured "
+                "'filters:' list to set 'op', 'column', 'keep', etc."
+            )
     filters = data_source.get("filters")
     if filters is None:
         return

src/zagg/configs/atl03.yaml

@@ -0,0 +1,95 @@
+# ATL03 photon-height aggregation template.
+#
+# Scalar-only stats (non-scalar outputs are deferred to #29): per-cell density
+# (photon count), min/max elevation, mean and median height, and variance.
+#
+# Confidence filter drops only TEP (transmit-echo path) photons and keeps
+# everything else: signal_conf_ph != -2 (column 0, the land surface type), per
+# the ATL03 v3 data dictionary
+# (https://nsidc.org/sites/default/files/icesat2_atl03_data_dict_v003_0.pdf).
+#
+# signal_conf_ph is 2-D (n_photons x 5 surface types) and TEP is flagged -2
+# uniformly across every surface type, so the land column (0) is operationally
+# equivalent to any other column for the TEP drop. The structured filter list
+# replaces the older flat ``quality_filter`` form to access the ``column`` and
+# ``op: ne`` knobs (issue #43, Phase A). Grid is rectilinear (HEALPix order-19,
+# the ~10 m match, waits on mortie #35).
+#
+# Multi-level form (issue #43 Phase B + C) is declared so the read path can
+# bound base-rate IO via the coarse ``segments`` rep-point coords + the
+# ``ph_index_beg``/``segment_ph_cnt`` link. Without this, ``lat_ph``/``lon_ph``
+# would be full-read (~245 MB per beam) before any spatial filter, which is the
+# dominant driver of the Lambda OOMs on ATL03 region runs (#43 motivation).
+data_source:
+  reader: h5coro
+  driver: s3
+  groups: [gt1l, gt1r, gt2l, gt2r, gt3l, gt3r]
+  coordinates:
+    latitude: "/{group}/heights/lat_ph"
+    longitude: "/{group}/heights/lon_ph"
+  variables:
+    h_ph: "/{group}/heights/h_ph"
+  filters:
+    - dataset: "/{group}/heights/signal_conf_ph"
+      column: 0                  # land surface type; TEP is uniform across columns
+      op: ne
+      value: -2
+  base_level: photons
+  # ``coordinates`` uses bare names joined to the level's ``path``; link
+  # ``index_beg`` / ``count`` use absolute path templates so they remain
+  # readable when the link arrays live outside the level's path (the read
+  # path resolves both forms -- see ``_level_coord_paths`` in processing.py).
+  # ``variables`` at the level scope is documentation only; the read path
+  # consults the top-level ``data_source.variables`` + filter datasets.
+  levels:
+    photons:
+      path: "/{group}/heights"
+      coordinates: {latitude: lat_ph, longitude: lon_ph}
+      link: null
+    segments:
+      path: "/{group}/geolocation"
+      coordinates: {latitude: reference_photon_lat, longitude: reference_photon_lon}
+      link:
+        to: photons
+        index_beg: "/{group}/geolocation/ph_index_beg"
+        count: "/{group}/geolocation/segment_ph_cnt"
+        index_base: 1            # ATL03 ph_index_beg is 1-based per the v3 dict
+  read_plan:
+    spatial_index: segments
+    pad: 1                       # one segment of padding on each side per #43
+
+aggregation:
+  variables:
+    count:
+      function: len
+      source: h_ph
+      dtype: int32
+      fill_value: 0
+    h_min:
+      function: min
+      source: h_ph
+      dtype: float32
+    h_max:
+      function: max
+      source: h_ph
+      dtype: float32
+    h_mean:
+      function: mean
+      source: h_ph
+      dtype: float32
+    h_median:
+      function: median
+      source: h_ph
+      dtype: float32
+    h_variance:
+      function: var
+      source: h_ph
+      dtype: float32
+
+output:
+  grid:
+    type: rectilinear
+    crs: EPSG:4326
+    resolution: 0.0001            # degrees (~10 m at the equator)
+    bounds: [-180, -90, 180, 90]
+    chunk_shape: [256, 256]

src/zagg/configs/atl03_waveform_counts.yaml

@@ -0,0 +1,83 @@
+# ATL03 per-cell waveform-counts template.
+#
+# Aggregates photon heights (h_ph) into a 128-bin fixed-width histogram at 2 m
+# vertical spacing, centred on the per-cell median of h_ph itself. Each bin
+# spans 2 m, giving a 256 m window [median(h_ph) - 128 m, median(h_ph) + 128 m).
+# Out-of-range photons are dropped (not counted). The companion scalar
+# bin_start records median(h_ph) so the absolute elevation of bin 0 can be
+# recovered downstream.
+#
+# Note (issue #30 thread): an earlier draft of this template tried to centre
+# on np.median(dem_h), but dem_h lives in /{group}/geophys_corr/dem_h at the
+# segment level (one value per ~100 photons), not the heights group; centring
+# on the cell's own median(h_ph) is more natural for a photon-height density
+# template and avoids the cross-group / resolution-mismatch read.
+#
+# Caveat: median(h_ph) is a *per-cell* anchor, not a stable absolute reference.
+# In a bimodal cell (e.g. ground + canopy returns) the median can land between
+# the two clusters, splitting both into wings of the histogram; that is a real
+# tradeoff of centring on the photon distribution itself. bin_start records the
+# centring value so the bin-0 absolute elevation = (bin_start - 128 m) is still
+# recoverable, but comparing waveforms across cells in the same absolute frame
+# requires shifting each cell's bins by its own bin_start.
+#
+# Confidence filter is the same as atl03.yaml: drop only TEP photons by the
+# structured ``filters:`` list (signal_conf_ph[:, 0] != -2; TEP is uniform
+# across surface types). Multi-level form + ``read_plan`` also identical to
+# atl03.yaml so the planned-IO benefits (#43 Phase C) apply to the waveform
+# template equally. Grid is rectilinear, matching atl03.yaml.
+data_source:
+  reader: h5coro
+  driver: s3
+  groups: [gt1l, gt1r, gt2l, gt2r, gt3l, gt3r]
+  coordinates:
+    latitude: "/{group}/heights/lat_ph"
+    longitude: "/{group}/heights/lon_ph"
+  variables:
+    h_ph: "/{group}/heights/h_ph"
+  filters:
+    - dataset: "/{group}/heights/signal_conf_ph"
+      column: 0                  # land surface type; TEP is uniform across columns
+      op: ne
+      value: -2
+  base_level: photons
+  # See atl03.yaml for the rationale behind the bare-name coordinates +
+  # absolute link path templates + documentation-only level ``variables``.
+  levels:
+    photons:
+      path: "/{group}/heights"
+      coordinates: {latitude: lat_ph, longitude: lon_ph}
+      link: null
+    segments:
+      path: "/{group}/geolocation"
+      coordinates: {latitude: reference_photon_lat, longitude: reference_photon_lon}
+      link:
+        to: photons
+        index_beg: "/{group}/geolocation/ph_index_beg"
+        count: "/{group}/geolocation/segment_ph_cnt"
+        index_base: 1            # ATL03 ph_index_beg is 1-based per the v3 dict
+  read_plan:
+    spatial_index: segments
+    pad: 1                       # one segment of padding on each side per #43
+
+aggregation:
+  variables:
+    waveform_counts:
+      kind: vector
+      trailing_shape: 128
+      expression: "np.histogram(h_ph - np.median(h_ph), 128, (-128.0, 128.0))[0].astype('uint32')"
+      source: h_ph
+      dtype: uint32
+      fill_value: 0
+    bin_start:
+      function: median
+      source: h_ph
+      dtype: float32
+
+output:
+  grid:
+    type: rectilinear
+    crs: EPSG:4326
+    resolution: 0.0001            # degrees (~10 m at the equator)
+    bounds: [-180, -90, 180, 90]
+    chunk_shape: [256, 256]