Add a GeoParquet map source (stacked on #166 EF5 floodmaps)

docs/source/maps/source_tab.rst

@@ -246,6 +246,70 @@ The GeoTIFF source overlays a Cloud-Optimized GeoTIFF (COG) on the map at its na
 
 ------------------------------------------------------------------------------------------------------------------------
 
+++++
+Zarr
+++++
+
+The Zarr source renders a raster layer from a public `Zarr <https://zarr.dev/>`_ store with no pre-processing. TethysDash reads the chosen variable slice on demand, converts it to a Cloud-Optimized GeoTIFF in memory, and draws it — so you supply a store URL and a variable rather than a prepared COG. It is styled and queried like a GeoTIFF layer: pick a color ramp on the :ref:`style_tab` and click the map for pixel values.
+
+**Layer Properties:**
+    - **url:** (required) Public URL of the Zarr store (an ``https`` bucket or ``s3://`` URL).
+    - **variable:** (required) Array name to read (e.g. ``depth``).
+    - **index:** (optional) Slice index along the store's leading dimension for stacked ``[n, y, x]`` data (default ``0``). Bind it to a :ref:`variable input <variableinputs>` with ``${Variable Name}`` to switch slices on the fly — for example, drive it with a slider to animate.
+    - **mask_below:** (optional) Sample values at or below this number render transparent. Leave blank to use the store's own threshold, if it declares one.
+
+**Example JSON Configuration:**
+
+::
+
+    {
+        "type": "WebGLTile",
+        "props": {
+            "name": "Flood Depth",
+            "source": {
+                "type": "Zarr",
+                "props": {
+                    "url": "https://example.com/floodmaps.zarr",
+                    "variable": "depth",
+                    "index": "${Storm}"
+                }
+            }
+        }
+    }
+
+------------------------------------------------------------------------------------------------------------------------
+
+++++++++++
+GeoParquet
+++++++++++
+
+The GeoParquet source renders a **vector** layer from a public `GeoParquet <https://geoparquet.org/>`_ file. TethysDash reads the file on demand, converts it to GeoJSON (reprojected to EPSG:4326) in memory, and draws it — so you supply a file URL rather than a prepared GeoJSON. It is styled and queried like a GeoJSON layer.
+
+**Layer Properties:**
+    - **url:** (required) Public URL of the GeoParquet file (an ``https`` bucket or ``s3://`` URL).
+
+.. note::
+    The whole file is converted to GeoJSON and sent to the browser, so this suits moderate feature counts; very large files may be slow to load.
+
+**Example JSON Configuration:**
+
+::
+
+    {
+        "type": "VectorLayer",
+        "props": {
+            "name": "Buildings",
+            "source": {
+                "type": "GeoParquet",
+                "props": {
+                    "url": "https://example.com/buildings.parquet"
+                }
+            }
+        }
+    }
+
+------------------------------------------------------------------------------------------------------------------------
+
 +++++++++++++
 Custom Layers
 +++++++++++++

package.json

@@ -43,6 +43,7 @@
     "dompurify": "^3.1.6",
     "dotenv": "^16.0.1",
     "file-loader": "^6.2.0",
+    "geotiff": "2.1.3",
     "html-react-parser": "^5.1.18",
     "html2canvas": "^1.4.1",
     "json5": "^2.2.3",

pyproject.toml

@@ -34,7 +34,15 @@ dependencies = [
     "psycopg2-binary==2.9.11",
     "geoalchemy2==0.18.4",
     "tethys-platform>=4.5.0",
-    "python-dateutil"
+    "python-dateutil",
+    "numpy",
+    "zarr>=3.0",
+    "fsspec",
+    "aiohttp",
+    "rasterio>=1.3",
+    "rio-cogeo>=5.0",
+    "geopandas",
+    "pyarrow"
 ]
 classifiers = [
      "Environment :: Web Environment",

reactapp/__tests__/components/inputs/custom/SliderMetadata.test.js

@@ -595,3 +595,46 @@ test("SliderMetadata includes alignSteps and alignOffset in onChange when enable
     }),
   );
 });
+
+test("Number slider defaults Output Format to {{n}} when left empty", async () => {
+  const mockOnChange = jest.fn();
+
+  render(
+    <DataViewerModeContext.Provider value={{ inDataViewerMode: false }}>
+      <VariableInputsContext.Provider value={{ variableInputValues: {} }}>
+        <SliderMetadata
+          onChange={mockOnChange}
+          values={{}}
+          visualizationRef={null}
+        />
+      </VariableInputsContext.Provider>
+    </DataViewerModeContext.Provider>,
+  );
+
+  await selectEvent.select(screen.getByLabelText("Data Type Input"), "Number");
+  fireEvent.change(screen.getByLabelText("Minimum Input"), {
+    target: { value: "0" },
+  });
+  fireEvent.change(screen.getByLabelText("Maximum Input"), {
+    target: { value: "100" },
+  });
+  fireEvent.change(screen.getByLabelText("Step Input"), {
+    target: { value: "1" },
+  });
+
+  selectEvent.openMenu(screen.getByLabelText("Initial Value"));
+  fireEvent.click(await screen.findByText("50"));
+
+  // Output Format was never typed by the user -> defaulted to {{n}}, so the
+  // metadata is emitted and the save gate isn't blocked.
+  expect(mockOnChange).toHaveBeenLastCalledWith(
+    expect.objectContaining({
+      dataType: "Number",
+      min: 0,
+      max: 100,
+      step: 1,
+      initialValue: 50,
+      outputFormat: "{{n}}",
+    }),
+  );
+});

reactapp/__tests__/components/map/Map.test.js

@@ -1260,6 +1260,89 @@ test("Double-buffering done() is idempotent when called twice", async () => {
   jest.useRealTimers();
 });
 
+const tileLayer = (url) => [
+  {
+    type: "WebGLTile",
+    props: {
+      source: { type: "Image Tile", props: { url } },
+      name: "buf_layer",
+      zIndex: 0,
+    },
+  },
+];
+
+const wrapLayers = (layers) => (
+  <VariableInputsContext.Provider value={{ setVariableInputValues: jest.fn() }}>
+    <MapContextProvider>
+      <TestingComponent mapProps={{ layers }} />
+    </MapContextProvider>
+  </VariableInputsContext.Provider>
+);
+
+test("replacement tile layer stays hidden until painted, then reveals on swap", async () => {
+  const addLayerSpy = jest.spyOn(Map.prototype, "addLayer");
+  const removeLayerSpy = jest.spyOn(Map.prototype, "removeLayer");
+
+  const { rerender } = render(
+    wrapLayers(tileLayer("https://example.com/a/{z}/{y}/{x}")),
+  );
+  expect(await screen.findByText("Map Ready")).toBeInTheDocument();
+  await waitFor(() => expect(addLayerSpy.mock.calls.length).toBe(1));
+
+  rerender(wrapLayers(tileLayer("https://example.com/b/{z}/{y}/{x}")));
+  await waitFor(() => expect(addLayerSpy.mock.calls.length).toBe(2));
+
+  const newLayer = addLayerSpy.mock.calls[1][0];
+  // Hidden via opacity (still loading) while the old layer is still present.
+  expect(newLayer.getOpacity()).toBe(0);
+  expect(removeLayerSpy.mock.calls.length).toBe(0);
+
+  newLayer.getSource().dispatchEvent("tileloadend");
+
+  await waitFor(() => expect(removeLayerSpy.mock.calls.length).toBe(1));
+  // Revealed on the swap; the old layer is the one removed.
+  expect(newLayer.getOpacity()).toBe(1);
+  expect(removeLayerSpy.mock.calls[0][0].values_.name).toBe("buf_layer");
+});
+
+test("superseded frame is discarded; only the newest replacement reveals", async () => {
+  const addLayerSpy = jest.spyOn(Map.prototype, "addLayer");
+  const removeLayerSpy = jest.spyOn(Map.prototype, "removeLayer");
+
+  const { rerender } = render(
+    wrapLayers(tileLayer("https://example.com/a/{z}/{y}/{x}")),
+  );
+  expect(await screen.findByText("Map Ready")).toBeInTheDocument();
+  await waitFor(() => expect(addLayerSpy.mock.calls.length).toBe(1));
+
+  // Two rerenders before any tiles paint: the middle frame is superseded.
+  rerender(wrapLayers(tileLayer("https://example.com/b/{z}/{y}/{x}")));
+  await waitFor(() => expect(addLayerSpy.mock.calls.length).toBe(2));
+  rerender(wrapLayers(tileLayer("https://example.com/c/{z}/{y}/{x}")));
+  await waitFor(() => expect(addLayerSpy.mock.calls.length).toBe(3));
+
+  const layerA = addLayerSpy.mock.calls[0][0];
+  const layerB = addLayerSpy.mock.calls[1][0];
+  const layerC = addLayerSpy.mock.calls[2][0];
+
+  // Both replacements are still hidden (opacity 0); the original is untouched.
+  expect(layerA.getOpacity()).toBe(1);
+  expect(layerB.getOpacity()).toBe(0);
+  expect(layerC.getOpacity()).toBe(0);
+
+  // The newest frame paints first: it reveals and the old layer is dropped.
+  layerC.getSource().dispatchEvent("tileloadend");
+  await waitFor(() => expect(layerC.getOpacity()).toBe(1));
+  expect(removeLayerSpy.mock.calls.map((c) => c[0])).toContain(layerA);
+
+  // The superseded middle frame paints late: discarded, never revealed.
+  layerB.getSource().dispatchEvent("tileloadend");
+  await waitFor(() =>
+    expect(removeLayerSpy.mock.calls.map((c) => c[0])).toContain(layerB),
+  );
+  expect(layerB.getOpacity()).toBe(0);
+});
+
 test("GeoTIFF with empty sources is silently skipped (not a failed layer)", async () => {
   const addLayerSpy = jest.spyOn(Map.prototype, "addLayer");
   const layers = [

reactapp/__tests__/components/map/ModuleLoader.test.js

@@ -11,6 +11,8 @@ import moduleLoader, {
   loadESRIJSON,
   buildPolygonFill,
   withAntimeridianFix,
+  zarrSourceToGeoTIFF,
+  geoParquetToGeoJSON,
 } from "components/map/ModuleLoader";
 import WebGLTile from "ol/layer/WebGLTile.js";
 import ImageLayer from "ol/layer/Image.js";
@@ -1959,3 +1961,49 @@ describe("withAntimeridianFix", () => {
     );
   });
 });
+
+describe("zarrSourceToGeoTIFF", () => {
+  test("assembles the zarr/cog endpoint URL from the source fields", () => {
+    const out = zarrSourceToGeoTIFF({
+      type: "Zarr",
+      props: { url: "https://x/store.zarr", variable: "depth", index: "150" },
+    });
+    expect(out.type).toBe("GeoTIFF");
+    expect(out.props.normalize).toBe(true);
+    const url = out.props.sources[0].url;
+    expect(url).toContain("/apps/tethysdash/zarr/cog/?");
+    expect(url).toContain("src=https%3A%2F%2Fx%2Fstore.zarr");
+    expect(url).toContain("variable=depth");
+    expect(url).toContain("index=150");
+    expect(url).not.toContain("mask_below");
+  });
+
+  test("defaults index to 0 and includes mask_below when provided", () => {
+    const out = zarrSourceToGeoTIFF({
+      type: "Zarr",
+      props: { url: "https://x", variable: "t", mask_below: "0.5" },
+    });
+    const url = out.props.sources[0].url;
+    expect(url).toContain("index=0");
+    expect(url).toContain("mask_below=0.5");
+  });
+});
+
+describe("geoParquetToGeoJSON", () => {
+  test("assembles the geoparquet/geojson endpoint URL from the source url", () => {
+    const out = geoParquetToGeoJSON({
+      type: "GeoParquet",
+      props: { url: "https://x/data.parquet" },
+    });
+    expect(out.type).toBe("GeoJSON");
+    expect(out.props).toEqual({});
+    expect(out.geojson).toContain("/apps/tethysdash/geoparquet/geojson/?");
+    expect(out.geojson).toContain("src=https%3A%2F%2Fx%2Fdata.parquet");
+  });
+
+  test("handles a missing url without throwing", () => {
+    const out = geoParquetToGeoJSON({ type: "GeoParquet", props: {} });
+    expect(out.type).toBe("GeoJSON");
+    expect(out.geojson).toContain("geoparquet/geojson/?src=");
+  });
+});

reactapp/__tests__/components/map/geoTIFFStyle.test.js

@@ -125,7 +125,18 @@ describe("buildGeoTIFFStyleColor", () => {
         rampMin: "not-a-number",
         rampMax: 100,
       }),
-    ).toThrow(/finite numbers/);
+    ).toThrow(/both be set or both empty/);
+  });
+
+  test("empty rampMin and rampMax build a normalized [0,1] interpolate", () => {
+    const expr = buildGeoTIFFStyleColor({
+      rampName: "viridis",
+      rampMin: "",
+      rampMax: "",
+    });
+    expect(expr[0]).toBe("interpolate");
+    expect(expr[3]).toBe(0); // first stop at 0
+    expect(expr[expr.length - 2]).toBe(1); // last stop at 1
   });
 
   test("hasNodata wraps the interpolate in a `case` against band 2 with a transparent fallback", () => {
@@ -157,7 +168,7 @@ describe("buildGeoTIFFStyleColor", () => {
         rampMin: 0,
         rampMax: "",
       }),
-    ).toThrow(/finite numbers/);
+    ).toThrow(/both be set or both empty/);
   });
 
   test("treats an empty-string rampMin as NaN (covers the minIsEmpty true branch)", () => {
@@ -170,7 +181,7 @@ describe("buildGeoTIFFStyleColor", () => {
         rampMin: "",
         rampMax: 100,
       }),
-    ).toThrow(/finite numbers/);
+    ).toThrow(/both be set or both empty/);
   });
 
   test("steps === 1 short-circuits to t=0 (single-entry ramp covers the steps===1 branch)", () => {

reactapp/__tests__/components/map/utilities.test.js

@@ -1922,6 +1922,31 @@ test("queryLayerFeatures GeoTIFF returns band values at pixel", async () => {
   expect(features[0].geometry).toEqual({ type: "Point", coordinates: [0, 0] });
 });
 
+test("queryLayerFeatures Zarr dispatches to GeoTIFF pixel extraction", async () => {
+  const { map } = mockGeoTIFFMap({ getDataReturn: new Float32Array([42.5]) });
+  const zarrConfig = {
+    configuration: {
+      type: "WebGLTile",
+      props: {
+        name: "Test GeoTIFF Layer",
+        source: {
+          type: "Zarr",
+          props: { url: "https://x", variable: "depth" },
+        },
+      },
+    },
+  };
+  const features = await queryLayerFeatures(
+    zarrConfig,
+    map,
+    [0, 0],
+    [400, 300],
+  );
+
+  expect(features).toHaveLength(1);
+  expect(features[0].attributes["Band 1"]).toBeCloseTo(42.5, 4);
+});
+
 test("queryLayerFeatures GeoTIFF reports multi-band values", async () => {
   const { map } = mockGeoTIFFMap({
     getDataReturn: new Uint8Array([12, 34, 56]),