diff --git a/docs/source/maps/source_tab.rst b/docs/source/maps/source_tab.rst index 58fe3c91..b7344bf5 100644 --- a/docs/source/maps/source_tab.rst +++ b/docs/source/maps/source_tab.rst @@ -246,6 +246,70 @@ The GeoTIFF source overlays a Cloud-Optimized GeoTIFF (COG) on the map at its na ------------------------------------------------------------------------------------------------------------------------ +++++ +Zarr +++++ + +The Zarr source renders a raster layer from a public `Zarr `_ store with no pre-processing. TethysDash reads the chosen variable slice on demand, converts it to a Cloud-Optimized GeoTIFF in memory, and draws it — so you supply a store URL and a variable rather than a prepared COG. It is styled and queried like a GeoTIFF layer: pick a color ramp on the :ref:`style_tab` and click the map for pixel values. + +**Layer Properties:** + - **url:** (required) Public URL of the Zarr store (an ``https`` bucket or ``s3://`` URL). + - **variable:** (required) Array name to read (e.g. ``depth``). + - **index:** (optional) Slice index along the store's leading dimension for stacked ``[n, y, x]`` data (default ``0``). Bind it to a :ref:`variable input ` with ``${Variable Name}`` to switch slices on the fly — for example, drive it with a slider to animate. + - **mask_below:** (optional) Sample values at or below this number render transparent. Leave blank to use the store's own threshold, if it declares one. + +**Example JSON Configuration:** + +:: + + { + "type": "WebGLTile", + "props": { + "name": "Flood Depth", + "source": { + "type": "Zarr", + "props": { + "url": "https://example.com/floodmaps.zarr", + "variable": "depth", + "index": "${Storm}" + } + } + } + } + +------------------------------------------------------------------------------------------------------------------------ + +++++++++++ +GeoParquet +++++++++++ + +The GeoParquet source renders a **vector** layer from a public `GeoParquet `_ file. TethysDash reads the file on demand, converts it to GeoJSON (reprojected to EPSG:4326) in memory, and draws it — so you supply a file URL rather than a prepared GeoJSON. It is styled and queried like a GeoJSON layer. + +**Layer Properties:** + - **url:** (required) Public URL of the GeoParquet file (an ``https`` bucket or ``s3://`` URL). + +.. note:: + The whole file is converted to GeoJSON and sent to the browser, so this suits moderate feature counts; very large files may be slow to load. + +**Example JSON Configuration:** + +:: + + { + "type": "VectorLayer", + "props": { + "name": "Buildings", + "source": { + "type": "GeoParquet", + "props": { + "url": "https://example.com/buildings.parquet" + } + } + } + } + +------------------------------------------------------------------------------------------------------------------------ + +++++++++++++ Custom Layers +++++++++++++ diff --git a/package-lock.json b/package-lock.json index cdb6cc64..28f92993 100644 --- a/package-lock.json +++ b/package-lock.json @@ -33,6 +33,7 @@ "dompurify": "^3.1.6", "dotenv": "^16.0.1", "file-loader": "^6.2.0", + "geotiff": "2.1.3", "html-react-parser": "^5.1.18", "html2canvas": "^1.4.1", "json5": "^2.2.3", diff --git a/package.json b/package.json index 12b3d77c..198f2531 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "dompurify": "^3.1.6", "dotenv": "^16.0.1", "file-loader": "^6.2.0", + "geotiff": "2.1.3", "html-react-parser": "^5.1.18", "html2canvas": "^1.4.1", "json5": "^2.2.3", diff --git a/pyproject.toml b/pyproject.toml index e41e632e..cf63717b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,15 @@ dependencies = [ "psycopg2-binary==2.9.11", "geoalchemy2==0.18.4", "tethys-platform>=4.5.0", - "python-dateutil" + "python-dateutil", + "numpy", + "zarr>=3.0", + "fsspec", + "aiohttp", + "rasterio>=1.3", + "rio-cogeo>=5.0", + "geopandas", + "pyarrow" ] classifiers = [ "Environment :: Web Environment", diff --git a/reactapp/__tests__/components/inputs/custom/SliderMetadata.test.js b/reactapp/__tests__/components/inputs/custom/SliderMetadata.test.js index 0810a12b..59746d0d 100644 --- a/reactapp/__tests__/components/inputs/custom/SliderMetadata.test.js +++ b/reactapp/__tests__/components/inputs/custom/SliderMetadata.test.js @@ -595,3 +595,46 @@ test("SliderMetadata includes alignSteps and alignOffset in onChange when enable }), ); }); + +test("Number slider defaults Output Format to {{n}} when left empty", async () => { + const mockOnChange = jest.fn(); + + render( + + + + + , + ); + + await selectEvent.select(screen.getByLabelText("Data Type Input"), "Number"); + fireEvent.change(screen.getByLabelText("Minimum Input"), { + target: { value: "0" }, + }); + fireEvent.change(screen.getByLabelText("Maximum Input"), { + target: { value: "100" }, + }); + fireEvent.change(screen.getByLabelText("Step Input"), { + target: { value: "1" }, + }); + + selectEvent.openMenu(screen.getByLabelText("Initial Value")); + fireEvent.click(await screen.findByText("50")); + + // Output Format was never typed by the user -> defaulted to {{n}}, so the + // metadata is emitted and the save gate isn't blocked. + expect(mockOnChange).toHaveBeenLastCalledWith( + expect.objectContaining({ + dataType: "Number", + min: 0, + max: 100, + step: 1, + initialValue: 50, + outputFormat: "{{n}}", + }), + ); +}); diff --git a/reactapp/__tests__/components/map/Map.test.js b/reactapp/__tests__/components/map/Map.test.js index 33b45689..855100f1 100644 --- a/reactapp/__tests__/components/map/Map.test.js +++ b/reactapp/__tests__/components/map/Map.test.js @@ -1260,6 +1260,89 @@ test("Double-buffering done() is idempotent when called twice", async () => { jest.useRealTimers(); }); +const tileLayer = (url) => [ + { + type: "WebGLTile", + props: { + source: { type: "Image Tile", props: { url } }, + name: "buf_layer", + zIndex: 0, + }, + }, +]; + +const wrapLayers = (layers) => ( + + + + + +); + +test("replacement tile layer stays hidden until painted, then reveals on swap", async () => { + const addLayerSpy = jest.spyOn(Map.prototype, "addLayer"); + const removeLayerSpy = jest.spyOn(Map.prototype, "removeLayer"); + + const { rerender } = render( + wrapLayers(tileLayer("https://example.com/a/{z}/{y}/{x}")), + ); + expect(await screen.findByText("Map Ready")).toBeInTheDocument(); + await waitFor(() => expect(addLayerSpy.mock.calls.length).toBe(1)); + + rerender(wrapLayers(tileLayer("https://example.com/b/{z}/{y}/{x}"))); + await waitFor(() => expect(addLayerSpy.mock.calls.length).toBe(2)); + + const newLayer = addLayerSpy.mock.calls[1][0]; + // Hidden via opacity (still loading) while the old layer is still present. + expect(newLayer.getOpacity()).toBe(0); + expect(removeLayerSpy.mock.calls.length).toBe(0); + + newLayer.getSource().dispatchEvent("tileloadend"); + + await waitFor(() => expect(removeLayerSpy.mock.calls.length).toBe(1)); + // Revealed on the swap; the old layer is the one removed. + expect(newLayer.getOpacity()).toBe(1); + expect(removeLayerSpy.mock.calls[0][0].values_.name).toBe("buf_layer"); +}); + +test("superseded frame is discarded; only the newest replacement reveals", async () => { + const addLayerSpy = jest.spyOn(Map.prototype, "addLayer"); + const removeLayerSpy = jest.spyOn(Map.prototype, "removeLayer"); + + const { rerender } = render( + wrapLayers(tileLayer("https://example.com/a/{z}/{y}/{x}")), + ); + expect(await screen.findByText("Map Ready")).toBeInTheDocument(); + await waitFor(() => expect(addLayerSpy.mock.calls.length).toBe(1)); + + // Two rerenders before any tiles paint: the middle frame is superseded. + rerender(wrapLayers(tileLayer("https://example.com/b/{z}/{y}/{x}"))); + await waitFor(() => expect(addLayerSpy.mock.calls.length).toBe(2)); + rerender(wrapLayers(tileLayer("https://example.com/c/{z}/{y}/{x}"))); + await waitFor(() => expect(addLayerSpy.mock.calls.length).toBe(3)); + + const layerA = addLayerSpy.mock.calls[0][0]; + const layerB = addLayerSpy.mock.calls[1][0]; + const layerC = addLayerSpy.mock.calls[2][0]; + + // Both replacements are still hidden (opacity 0); the original is untouched. + expect(layerA.getOpacity()).toBe(1); + expect(layerB.getOpacity()).toBe(0); + expect(layerC.getOpacity()).toBe(0); + + // The newest frame paints first: it reveals and the old layer is dropped. + layerC.getSource().dispatchEvent("tileloadend"); + await waitFor(() => expect(layerC.getOpacity()).toBe(1)); + expect(removeLayerSpy.mock.calls.map((c) => c[0])).toContain(layerA); + + // The superseded middle frame paints late: discarded, never revealed. + layerB.getSource().dispatchEvent("tileloadend"); + await waitFor(() => + expect(removeLayerSpy.mock.calls.map((c) => c[0])).toContain(layerB), + ); + expect(layerB.getOpacity()).toBe(0); +}); + test("GeoTIFF with empty sources is silently skipped (not a failed layer)", async () => { const addLayerSpy = jest.spyOn(Map.prototype, "addLayer"); const layers = [ diff --git a/reactapp/__tests__/components/map/ModuleLoader.test.js b/reactapp/__tests__/components/map/ModuleLoader.test.js index 030a1e6e..126f84fe 100644 --- a/reactapp/__tests__/components/map/ModuleLoader.test.js +++ b/reactapp/__tests__/components/map/ModuleLoader.test.js @@ -11,6 +11,8 @@ import moduleLoader, { loadESRIJSON, buildPolygonFill, withAntimeridianFix, + zarrSourceToGeoTIFF, + geoParquetToGeoJSON, } from "components/map/ModuleLoader"; import WebGLTile from "ol/layer/WebGLTile.js"; import ImageLayer from "ol/layer/Image.js"; @@ -1959,3 +1961,49 @@ describe("withAntimeridianFix", () => { ); }); }); + +describe("zarrSourceToGeoTIFF", () => { + test("assembles the zarr/cog endpoint URL from the source fields", () => { + const out = zarrSourceToGeoTIFF({ + type: "Zarr", + props: { url: "https://x/store.zarr", variable: "depth", index: "150" }, + }); + expect(out.type).toBe("GeoTIFF"); + expect(out.props.normalize).toBe(true); + const url = out.props.sources[0].url; + expect(url).toContain("/apps/tethysdash/zarr/cog/?"); + expect(url).toContain("src=https%3A%2F%2Fx%2Fstore.zarr"); + expect(url).toContain("variable=depth"); + expect(url).toContain("index=150"); + expect(url).not.toContain("mask_below"); + }); + + test("defaults index to 0 and includes mask_below when provided", () => { + const out = zarrSourceToGeoTIFF({ + type: "Zarr", + props: { url: "https://x", variable: "t", mask_below: "0.5" }, + }); + const url = out.props.sources[0].url; + expect(url).toContain("index=0"); + expect(url).toContain("mask_below=0.5"); + }); +}); + +describe("geoParquetToGeoJSON", () => { + test("assembles the geoparquet/geojson endpoint URL from the source url", () => { + const out = geoParquetToGeoJSON({ + type: "GeoParquet", + props: { url: "https://x/data.parquet" }, + }); + expect(out.type).toBe("GeoJSON"); + expect(out.props).toEqual({}); + expect(out.geojson).toContain("/apps/tethysdash/geoparquet/geojson/?"); + expect(out.geojson).toContain("src=https%3A%2F%2Fx%2Fdata.parquet"); + }); + + test("handles a missing url without throwing", () => { + const out = geoParquetToGeoJSON({ type: "GeoParquet", props: {} }); + expect(out.type).toBe("GeoJSON"); + expect(out.geojson).toContain("geoparquet/geojson/?src="); + }); +}); diff --git a/reactapp/__tests__/components/map/geoTIFFStyle.test.js b/reactapp/__tests__/components/map/geoTIFFStyle.test.js index 3fc5b453..14a315a3 100644 --- a/reactapp/__tests__/components/map/geoTIFFStyle.test.js +++ b/reactapp/__tests__/components/map/geoTIFFStyle.test.js @@ -125,7 +125,18 @@ describe("buildGeoTIFFStyleColor", () => { rampMin: "not-a-number", rampMax: 100, }), - ).toThrow(/finite numbers/); + ).toThrow(/both be set or both empty/); + }); + + test("empty rampMin and rampMax build a normalized [0,1] interpolate", () => { + const expr = buildGeoTIFFStyleColor({ + rampName: "viridis", + rampMin: "", + rampMax: "", + }); + expect(expr[0]).toBe("interpolate"); + expect(expr[3]).toBe(0); // first stop at 0 + expect(expr[expr.length - 2]).toBe(1); // last stop at 1 }); test("hasNodata wraps the interpolate in a `case` against band 2 with a transparent fallback", () => { @@ -157,7 +168,7 @@ describe("buildGeoTIFFStyleColor", () => { rampMin: 0, rampMax: "", }), - ).toThrow(/finite numbers/); + ).toThrow(/both be set or both empty/); }); test("treats an empty-string rampMin as NaN (covers the minIsEmpty true branch)", () => { @@ -170,7 +181,7 @@ describe("buildGeoTIFFStyleColor", () => { rampMin: "", rampMax: 100, }), - ).toThrow(/finite numbers/); + ).toThrow(/both be set or both empty/); }); test("steps === 1 short-circuits to t=0 (single-entry ramp covers the steps===1 branch)", () => { diff --git a/reactapp/__tests__/components/map/utilities.test.js b/reactapp/__tests__/components/map/utilities.test.js index 153c0b38..ee884b55 100644 --- a/reactapp/__tests__/components/map/utilities.test.js +++ b/reactapp/__tests__/components/map/utilities.test.js @@ -1922,6 +1922,31 @@ test("queryLayerFeatures GeoTIFF returns band values at pixel", async () => { expect(features[0].geometry).toEqual({ type: "Point", coordinates: [0, 0] }); }); +test("queryLayerFeatures Zarr dispatches to GeoTIFF pixel extraction", async () => { + const { map } = mockGeoTIFFMap({ getDataReturn: new Float32Array([42.5]) }); + const zarrConfig = { + configuration: { + type: "WebGLTile", + props: { + name: "Test GeoTIFF Layer", + source: { + type: "Zarr", + props: { url: "https://x", variable: "depth" }, + }, + }, + }, + }; + const features = await queryLayerFeatures( + zarrConfig, + map, + [0, 0], + [400, 300], + ); + + expect(features).toHaveLength(1); + expect(features[0].attributes["Band 1"]).toBeCloseTo(42.5, 4); +}); + test("queryLayerFeatures GeoTIFF reports multi-band values", async () => { const { map } = mockGeoTIFFMap({ getDataReturn: new Uint8Array([12, 34, 56]), diff --git a/reactapp/__tests__/components/modals/DataViewer/DataViewer.sliderSave.test.js b/reactapp/__tests__/components/modals/DataViewer/DataViewer.sliderSave.test.js new file mode 100644 index 00000000..0e105c4c --- /dev/null +++ b/reactapp/__tests__/components/modals/DataViewer/DataViewer.sliderSave.test.js @@ -0,0 +1,111 @@ +import { useEffect, useContext } from "react"; +import PropTypes from "prop-types"; +import { render, screen, fireEvent, waitFor } from "@testing-library/react"; +import DataViewerModal from "components/modals/DataViewer/DataViewer"; +import { + userDashboard, + mockedDateRangeVariable, +} from "__tests__/utilities/constants"; +import createLoadedComponent, { + InputVariablePComponent, +} from "__tests__/utilities/customRender"; +import { GridItemContext, TabContext } from "components/contexts/Contexts"; + +const { ResizeObserver } = window; + +beforeEach(() => { + delete window.ResizeObserver; + window.ResizeObserver = jest.fn().mockImplementation(() => ({ + observe: jest.fn(), + unobserve: jest.fn(), + disconnect: jest.fn(), + })); +}); + +afterEach(() => { + window.ResizeObserver = ResizeObserver; + jest.restoreAllMocks(); +}); + +const TestingComponent = ({ gridItem, onTabUpdate }) => { + const { tabs } = useContext(TabContext); + useEffect(() => { + onTabUpdate(tabs); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [tabs]); + return ( + + + + ); +}; + +TestingComponent.propTypes = { + gridItem: PropTypes.object, + onTabUpdate: PropTypes.func, +}; + +const sliderGridItem = () => { + const gridItem = JSON.parse(JSON.stringify(mockedDateRangeVariable)); + gridItem.args_string = JSON.stringify({ + variable_name: "Storm", + show_label: true, + variable_options_source: "slider", + "variable_options_source.metadata": { + min: 0, + max: 10, + step: 1, + dataType: "Number", + outputFormat: "{{n}}", + }, + initial_value: null, + }); + return gridItem; +}; + +test("a slider variable input with no explicit initial value saves without a false validation error", async () => { + const mockedDashboard = JSON.parse(JSON.stringify(userDashboard)); + const gridItem = sliderGridItem(); + mockedDashboard.tabs[0].gridItems[0] = gridItem; + const mockUpdateTab = jest.fn(); + + render( + createLoadedComponent({ + children: ( + <> + + + + ), + options: { initialDashboard: mockedDashboard }, + }), + ); + + const saveButton = await screen.findByLabelText("dataviewer-save-button"); + + mockUpdateTab.mockClear(); + fireEvent.click(saveButton); + + await waitFor(() => expect(mockUpdateTab).toHaveBeenCalled()); + expect( + screen.queryByText("Initial value must be selected in the dropdown"), + ).not.toBeInTheDocument(); + + const savedArgs = JSON.parse( + mockUpdateTab.mock.calls.at(-1)[0][0].gridItems[0].args_string, + ); + expect(savedArgs.variable_options_source).toBe("slider"); +}); diff --git a/reactapp/__tests__/components/modals/MapLayer/GeoTIFFSourceModal.test.js b/reactapp/__tests__/components/modals/MapLayer/GeoTIFFSourceModal.test.js index ab1320e1..6518c600 100644 --- a/reactapp/__tests__/components/modals/MapLayer/GeoTIFFSourceModal.test.js +++ b/reactapp/__tests__/components/modals/MapLayer/GeoTIFFSourceModal.test.js @@ -128,6 +128,7 @@ test("GeoTIFFSourceModal typing a URL and clicking Save calls onSave with all fi await user.type(screen.getByLabelText("Bands Input"), "1,2,3"); await user.type(screen.getByLabelText("Min Input"), "10"); await user.type(screen.getByLabelText("Max Input"), "255"); + await user.clear(screen.getByLabelText("Nodata Input")); await user.type(screen.getByLabelText("Nodata Input"), "0"); await user.type(screen.getByLabelText("Projection Input"), "EPSG:4326"); @@ -245,6 +246,7 @@ test("GeoTIFFSourceModal preserves string '0' in min on save (not coerced to num ); await user.type(screen.getByLabelText("Min Input"), "0"); await user.type(screen.getByLabelText("Max Input"), "0"); + await user.clear(screen.getByLabelText("Nodata Input")); await user.type(screen.getByLabelText("Nodata Input"), "0"); await user.click( @@ -395,3 +397,9 @@ test("GeoTIFFSourceModal returns focus to returnFocusRef element on close", asyn { timeout: 2000 }, ); }); + +test("new GeoTIFF source leaves Nodata empty (no assumed sentinel)", async () => { + render(); + expect(await screen.findByText("Add GeoTIFF Source")).toBeInTheDocument(); + expect(screen.getByLabelText("Nodata Input")).toHaveValue(""); +}); diff --git a/reactapp/__tests__/components/modals/MapLayer/MapLayer.test.js b/reactapp/__tests__/components/modals/MapLayer/MapLayer.test.js index 1af23db0..141ef500 100644 --- a/reactapp/__tests__/components/modals/MapLayer/MapLayer.test.js +++ b/reactapp/__tests__/components/modals/MapLayer/MapLayer.test.js @@ -2354,12 +2354,14 @@ describe("MapLayerModal GeoTIFF ramp-style save path (Unit 7)", () => { expect(savedStyle.color[3]).toBe(0); // Last stop pair ends at rampMax. expect(savedStyle.color[savedStyle.color.length - 2]).toBe(100); + // Explicit range = raw band values, so the source is not normalized. + expect(savedConfig.configuration.props.source.props.normalize).toBe(false); // Most important regression guard: the backend upload was NOT called. expect(uploadSpy).not.toHaveBeenCalled(); }); - test("GeoTIFF without a ramp name saves with no style key", async () => { + test("GeoTIFF with no explicit range gets a normalized style (turbo default)", async () => { const uploadSpy = jest .spyOn(appAPI, "uploadJSON") .mockResolvedValue({ success: true, filename: "x.json" }); @@ -2395,11 +2397,65 @@ describe("MapLayerModal GeoTIFF ramp-style save path (Unit 7)", () => { }); const savedConfig = addMapLayer.mock.calls[0][0]; - expect(savedConfig.configuration.style).toBeUndefined(); + const savedStyle = savedConfig.configuration.style; + // Turbo default + normalized [0,1] interpolate; no persisted range. + expect(Array.isArray(savedStyle.color)).toBe(true); + expect(savedStyle.color[0]).toBe("interpolate"); + expect(savedStyle.color[savedStyle.color.length - 2]).toBe(1); + expect(savedConfig.configuration.props.source.rampName).toBe("turbo"); + expect(savedConfig.configuration.props.source.rampMin).toBeUndefined(); + expect(savedConfig.configuration.props.source.props.normalize).toBe(true); expect(uploadSpy).not.toHaveBeenCalled(); }); - test("GeoTIFF with rampName but empty rampMin/rampMax does not generate a style", async () => { + test("Zarr source saves as a WebGLTile layer with a normalized turbo ramp", async () => { + const handleModalClose = jest.fn(); + const addMapLayer = jest.fn(); + const layerInfo = { + layerProps: { name: "Flood Depth" }, + sourceProps: { + type: "Zarr", + props: { + url: "https://x/store.zarr", + variable: "depth", + index: "5", + }, + }, + }; + + render( + , + ); + + fireEvent.click(await screen.findByLabelText("Create Layer Button")); + await waitFor(() => { + expect(addMapLayer).toHaveBeenCalledTimes(1); + }); + + const savedConfig = addMapLayer.mock.calls[0][0]; + // WebGLTile layer with the Zarr source and its fields preserved for editing. + expect(savedConfig.configuration.type).toBe("WebGLTile"); + const source = savedConfig.configuration.props.source; + expect(source.type).toBe("Zarr"); + expect(source.props.url).toBe("https://x/store.zarr"); + expect(source.props.variable).toBe("depth"); + expect(source.props.index).toBe("5"); + // Turbo default + per-slice auto-scaling (normalized ramp). Zarr COGs always + // carry a -9999 nodata sentinel, so the ramp is wrapped in a transparency + // `case` expression (a GeoTIFF with no nodata set would be bare interpolate). + const color = savedConfig.configuration.style.color; + expect(color[0]).toBe("case"); + expect(JSON.stringify(color)).toContain("interpolate"); + expect(source.rampName).toBe("turbo"); + expect(source.props.normalize).toBe(true); + }); + + test("GeoTIFF with rampName and empty range gets a normalized style", async () => { const uploadSpy = jest .spyOn(appAPI, "uploadJSON") .mockResolvedValue({ success: true, filename: "x.json" }); @@ -2407,7 +2463,7 @@ describe("MapLayerModal GeoTIFF ramp-style save path (Unit 7)", () => { const handleModalClose = jest.fn(); const addMapLayer = jest.fn(); const layerInfo = { - layerProps: { name: "Incomplete Ramp GeoTIFF" }, + layerProps: { name: "Auto Ramp GeoTIFF" }, sourceProps: { type: "GeoTIFF", rampName: "viridis", @@ -2438,7 +2494,12 @@ describe("MapLayerModal GeoTIFF ramp-style save path (Unit 7)", () => { }); const savedConfig = addMapLayer.mock.calls[0][0]; - expect(savedConfig.configuration.style).toBeUndefined(); + const savedStyle = savedConfig.configuration.style; + expect(Array.isArray(savedStyle.color)).toBe(true); + expect(savedStyle.color[0]).toBe("interpolate"); + expect(savedStyle.color[savedStyle.color.length - 2]).toBe(1); + expect(savedConfig.configuration.props.source.rampMin).toBeUndefined(); + expect(savedConfig.configuration.props.source.props.normalize).toBe(true); expect(uploadSpy).not.toHaveBeenCalled(); }); @@ -3780,6 +3841,14 @@ describe("getLayerType", () => { expect(getLayerType("GeoTIFF")).toBe("WebGLTile"); }); + test("Zarr maps to WebGLTile (renders as a COG)", () => { + expect(getLayerType("Zarr")).toBe("WebGLTile"); + }); + + test("GeoParquet falls through to VectorLayer", () => { + expect(getLayerType("GeoParquet")).toBe("VectorLayer"); + }); + test("Vector source types map to VectorTileLayer", () => { expect(getLayerType("Vector Tile")).toBe("VectorTileLayer"); expect(getLayerType("Vector")).toBe("VectorTileLayer"); diff --git a/reactapp/__tests__/components/modals/MapLayer/StylePane.test.js b/reactapp/__tests__/components/modals/MapLayer/StylePane.test.js index e4448f9a..616a5d5b 100644 --- a/reactapp/__tests__/components/modals/MapLayer/StylePane.test.js +++ b/reactapp/__tests__/components/modals/MapLayer/StylePane.test.js @@ -6,6 +6,9 @@ import PropTypes from "prop-types"; import userEvent from "@testing-library/user-event"; import { LayoutContext, AppContext } from "components/contexts/Contexts"; import * as utilities from "components/map/utilities"; +import { fromUrl } from "geotiff"; + +jest.mock("geotiff", () => ({ fromUrl: jest.fn() })); const exampleStyle = { version: 8, @@ -261,7 +264,12 @@ test("StylePane Updating Existing GeoJSON", async () => { test("StylePane Styling not available", async () => { render(); - const supportedTypes = ["GeoJSON", "ESRI Feature Service", "PMTiles Vector"]; + const supportedTypes = [ + "GeoJSON", + "ESRI Feature Service", + "PMTiles Vector", + "GeoParquet", + ]; expect( await screen.findByText( `Custom Styling is only available for ${supportedTypes.join(", ")} layers.`, @@ -269,6 +277,12 @@ test("StylePane Styling not available", async () => { ).toBeInTheDocument(); }); +test("StylePane offers vector styling for a GeoParquet source", async () => { + render(); + // Supported vector source -> styling controls, not the "not available" notice. + expect(await screen.findByLabelText("Rule-based Editor")).toBeInTheDocument(); +}); + test("StylePane switches to rules mode and syncs rules/defaultStyle from JSON", async () => { render(); // Switch to rules mode @@ -531,6 +545,91 @@ test("StylePane renders Color Ramp section for GeoTIFF source type", async () => expect(screen.getByLabelText("Ramp Max")).toBeInTheDocument(); }); +test("StylePane defaults a GeoTIFF source's ramp to turbo when none is set", async () => { + render(); + await waitFor(() => { + expect(screen.getByTestId("rampName")).toHaveTextContent("turbo"); + }); +}); + +test("StylePane renders the Color Ramp section and defaults to turbo for a Zarr source", async () => { + render(); + expect(await screen.findByText("Color Ramp")).toBeInTheDocument(); + await waitFor(() => { + expect(screen.getByTestId("rampName")).toHaveTextContent("turbo"); + }); +}); + +test("StylePane pre-fills GeoTIFF ramp min/max from statistics, and clearing sticks", async () => { + fromUrl.mockReset(); + fromUrl.mockResolvedValue({ + getImage: async () => ({ + getGDALMetadata: () => ({ + STATISTICS_MINIMUM: "0.05", + STATISTICS_MAXIMUM: "11.7", + }), + }), + }); + const user = userEvent.setup(); + + render( + , + ); + + await waitFor(() => + expect(screen.getByTestId("rampMax")).toHaveTextContent("12"), + ); + expect(screen.getByTestId("rampMin")).toHaveTextContent("0"); + expect(fromUrl).toHaveBeenCalledTimes(1); + + // Clearing the fields must not re-fetch/refill (opts into per-storm auto). + await user.clear(screen.getByLabelText("Ramp Min")); + await user.clear(screen.getByLabelText("Ramp Max")); + expect(fromUrl).toHaveBeenCalledTimes(1); +}); + +test("StylePane skips the stats fetch when ramp min/max are already set", async () => { + fromUrl.mockReset(); + + render( + , + ); + + await screen.findByText("Color Ramp"); + expect(fromUrl).not.toHaveBeenCalled(); +}); + +test("StylePane does not fetch stats for a non-http source URL", async () => { + fromUrl.mockReset(); + + render( + , + ); + + await screen.findByText("Color Ramp"); + expect(fromUrl).not.toHaveBeenCalled(); +}); + test("StylePane does NOT render Color Ramp section for non-GeoTIFF sources", async () => { render(); // Vector editor renders instead. diff --git a/reactapp/components/inputs/custom/SliderMetadata.js b/reactapp/components/inputs/custom/SliderMetadata.js index 4080871a..20b75b87 100644 --- a/reactapp/components/inputs/custom/SliderMetadata.js +++ b/reactapp/components/inputs/custom/SliderMetadata.js @@ -188,7 +188,8 @@ const SliderMetadata = ({ onChange, values }) => { setMax(null); setStep(null); setInitialValue(null); - setOutputFormat(""); + // Number sliders get a default format so Output Format isn't required. + setOutputFormat(selected?.value === "Number" ? "{{n}}" : ""); onChange(null); }; diff --git a/reactapp/components/map/Map.js b/reactapp/components/map/Map.js index 8779eb43..eeaeff57 100644 --- a/reactapp/components/map/Map.js +++ b/reactapp/components/map/Map.js @@ -75,8 +75,33 @@ const MapComponent = ({ const isFirstRender = useRef(true); const mapExtentVariableEvent = useRef(); const currentLayers = useRef([]); + const layerSyncToken = useRef(0); + const activeFadeRef = useRef(null); const { setVariableInputValues } = useContext(VariableInputsContext); + // Fade the incoming layers in over `duration` ms, then remove the outgoing + // ones, so a storm swap dissolves instead of flashing. Any running fade is + // finalized first so overlapping swaps don't leave a layer mid-fade. + const crossfadeLayers = (map, incoming, outgoing, duration) => { + if (activeFadeRef.current) activeFadeRef.current(); + const start = Date.now(); + let rafId = null; + const finalize = () => { + if (rafId !== null) cancelAnimationFrame(rafId); + incoming.forEach(({ layer, opacity }) => layer.setOpacity(opacity)); + outgoing.forEach((layer) => map.removeLayer(layer)); + activeFadeRef.current = null; + }; + const step = () => { + const t = Math.min(1, (Date.now() - start) / duration); + incoming.forEach(({ layer, opacity }) => layer.setOpacity(opacity * t)); + if (t < 1) rafId = requestAnimationFrame(step); + else finalize(); + }; + activeFadeRef.current = finalize; + rafId = requestAnimationFrame(step); + }; + const defaultMapConfig = { className: "ol-map", style: { width: "100%", height: "100%", position: "relative" }, @@ -123,6 +148,7 @@ const MapComponent = ({ return () => { // istanbul ignore next if (visualizationRef.current) { + if (activeFadeRef.current) activeFadeRef.current(); visualizationRef.current.setTarget(undefined); visualizationRef.current = null; } @@ -198,6 +224,9 @@ const MapComponent = ({ const updateLayers = async () => { const map = visualizationRef.current; const currentMapLayers = map.getLayers().getArray(); + // Identify this run so a newer frame can supersede it mid-load. + layerSyncToken.current += 1; + const myToken = layerSyncToken.current; // Clean up layers: determine which to keep and which to remove const layersToKeep = []; @@ -303,6 +332,8 @@ const MapComponent = ({ // setup constants for handling new layers const customLayers = layers ?? []; let failedLayers = []; + // Replacement layers added hidden until painted, then revealed on swap. + const buffered = []; // Add or update layers in parallel const layerLoadPromises = []; @@ -371,6 +402,14 @@ const MapComponent = ({ setTimeout(done, 5000); }); layerLoadPromises.push(loadPromise); + // Hide via opacity, not visibility: an invisible layer never + // renders, so it would never load its tiles. Opacity 0 keeps + // it loading; we restore the real opacity once it has painted. + buffered.push({ + layer: newLayer, + opacity: newLayer.getOpacity(), + }); + newLayer.setOpacity(0); } } @@ -378,7 +417,8 @@ const MapComponent = ({ if ( layerConfig.type === "WebGLTile" && - layerConfig.props?.source?.type === "GeoTIFF" + (layerConfig.props?.source?.type === "GeoTIFF" || + layerConfig.props?.source?.type === "Zarr") ) { const geoTIFFSource = newLayer.getSource(); @@ -536,10 +576,19 @@ const MapComponent = ({ await Promise.all(layerLoadPromises); } - // Remove layers that are no longer needed - layersToRemove.forEach((layer) => { - map.removeLayer(layer); - }); + // Reveal painted replacements, then drop old layers in one frame. + // A superseded run keeps the old layer and discards its unshown buffers, + // so fast playback skips frames instead of flashing or stalling. + const superseded = myToken !== layerSyncToken.current; + if (buffered.length > 0 && superseded) { + buffered.forEach(({ layer }) => map.removeLayer(layer)); + } else if (buffered.length > 0) { + crossfadeLayers(map, buffered, layersToRemove, 250); + } else { + layersToRemove.forEach((layer) => { + map.removeLayer(layer); + }); + } if (failedLayers.length > 0) { setErrorMessage( @@ -620,7 +669,11 @@ const MapComponent = ({ isFirstRender.current = false; } - currentLayers.current = layers ?? []; + // Only the winning run records the rendered layers, so a slow superseded + // run can't overwrite it with a stale config. + if (!superseded) { + currentLayers.current = layers ?? []; + } }; updateLayers(); diff --git a/reactapp/components/map/ModuleLoader.js b/reactapp/components/map/ModuleLoader.js index 03f58a36..62e28c0b 100644 --- a/reactapp/components/map/ModuleLoader.js +++ b/reactapp/components/map/ModuleLoader.js @@ -47,7 +47,53 @@ export function withAntimeridianFix(type, props) { }; } +const APP_ROOT_URL = process.env.TETHYS_APP_ROOT_URL ?? "/apps/tethysdash/"; + +// A "Zarr" source is sugar over the zarr/cog endpoint: the author supplies a +// store URL + variable (+ optional index/mask_below) and we assemble the COG +// URL, then render it as an ordinary GeoTIFF source. Variable inputs in the +// fields (e.g. index="${Storm}") are already substituted before this runs. +export function zarrSourceToGeoTIFF(config) { + const { url, variable, index, mask_below, normalize } = config.props ?? {}; + const params = new URLSearchParams({ + src: url ?? "", + variable: variable ?? "", + index: index ?? "0", + }); + if (mask_below !== undefined && mask_below !== "") { + params.set("mask_below", mask_below); + } + return { + ...config, + type: "GeoTIFF", + props: { + sources: [{ url: `${APP_ROOT_URL}zarr/cog/?${params.toString()}` }], + normalize: normalize ?? true, + }, + }; +} + +// A "GeoParquet" source is sugar over the geoparquet/geojson endpoint: the +// author supplies a file URL and we render the returned GeoJSON (EPSG:4326) as +// an ordinary vector source. +export function geoParquetToGeoJSON(config) { + const { url } = config.props ?? {}; + const params = new URLSearchParams({ src: url ?? "" }); + return { + ...config, + type: "GeoJSON", + geojson: `${APP_ROOT_URL}geoparquet/geojson/?${params.toString()}`, + props: {}, + }; +} + const moduleLoader = async (config, mapProjection) => { + if (config.type === "Zarr") { + config = zarrSourceToGeoTIFF(config); + } + if (config.type === "GeoParquet") { + config = geoParquetToGeoJSON(config); + } if ( config.type === "Static Image" && typeof config.props?.imageExtent === "string" @@ -187,7 +233,12 @@ const resolveProps = async (props, mapProjection) => { } } - if (props.sources && Array.isArray(props.sources)) { + if ( + props.sources && + Array.isArray(props.sources) && + props.normalize === undefined + ) { + // Default raw band values unless the layer explicitly asked to normalize. resolvedProps.normalize = false; } diff --git a/reactapp/components/map/geoTIFFStyle.js b/reactapp/components/map/geoTIFFStyle.js index 9d715125..5a730dac 100644 --- a/reactapp/components/map/geoTIFFStyle.js +++ b/reactapp/components/map/geoTIFFStyle.js @@ -12,18 +12,25 @@ export function buildGeoTIFFStyleColor({ throw new Error(`Unknown color ramp: ${rampName}`); } - // Reject empty strings up front — `Number("")` silently returns 0, which - // would otherwise pass the isFinite check and produce a degenerate expression - // that doesn't match the user's (missing) intent. - const minIsEmpty = typeof rampMin === "string" && rampMin.trim() === ""; - const maxIsEmpty = typeof rampMax === "string" && rampMax.trim() === ""; - const min = minIsEmpty ? NaN : Number(rampMin); - const max = maxIsEmpty ? NaN : Number(rampMax); - - if (!Number.isFinite(min) || !Number.isFinite(max)) { - throw new Error( - `rampMin and rampMax must be finite numbers (got rampMin=${rampMin}, rampMax=${rampMax})`, - ); + const minIsEmpty = + rampMin == null || (typeof rampMin === "string" && rampMin.trim() === ""); + const maxIsEmpty = + rampMax == null || (typeof rampMax === "string" && rampMax.trim() === ""); + + let min; + let max; + if (minIsEmpty && maxIsEmpty) { + // Normalized mode: OL scales band 1 to [0,1] from the file's statistics. + min = 0; + max = 1; + } else { + min = minIsEmpty ? NaN : Number(rampMin); + max = maxIsEmpty ? NaN : Number(rampMax); + if (!Number.isFinite(min) || !Number.isFinite(max)) { + throw new Error( + `rampMin and rampMax must both be set or both empty (got rampMin=${rampMin}, rampMax=${rampMax})`, + ); + } } const steps = colors.length; @@ -44,7 +51,7 @@ export function buildGeoTIFFStyleColor({ buildGeoTIFFStyleColor.propTypes = { rampName: PropTypes.string.isRequired, - rampMin: PropTypes.oneOfType([PropTypes.number, PropTypes.string]).isRequired, - rampMax: PropTypes.oneOfType([PropTypes.number, PropTypes.string]).isRequired, + rampMin: PropTypes.oneOfType([PropTypes.number, PropTypes.string]), + rampMax: PropTypes.oneOfType([PropTypes.number, PropTypes.string]), hasNodata: PropTypes.bool, }; diff --git a/reactapp/components/map/utilities.js b/reactapp/components/map/utilities.js index 43b21b7f..9fe572fc 100644 --- a/reactapp/components/map/utilities.js +++ b/reactapp/components/map/utilities.js @@ -18,7 +18,11 @@ import Protobuf from "pbf"; // Source types whose features live in a client-side OL VectorSource (vs // server-rendered services queried remotely). -export const CLIENT_VECTOR_SOURCE_TYPES = ["GeoJSON", "ESRI Feature Service"]; +export const CLIENT_VECTOR_SOURCE_TYPES = [ + "GeoJSON", + "ESRI Feature Service", + "GeoParquet", +]; // Coerce an optional numeric layer prop: GUI inputs emit strings, so accept // any numeric value but treat null/undefined/blank/non-numeric as unset. @@ -126,6 +130,23 @@ export const sourcePropertiesOptions = { required: {}, optional: {}, }, + Zarr: { + required: { + url: { placeholder: "Zarr store URL (https or s3 bucket)" }, + variable: { placeholder: "Variable / array name (e.g. depth)" }, + }, + optional: { + // eslint-disable-next-line no-template-curly-in-string + index: { placeholder: "Slice index or a variable, e.g. ${Storm}" }, + mask_below: { placeholder: "Mask values at or below this" }, + }, + }, + GeoParquet: { + required: { + url: { placeholder: "GeoParquet file URL (https or s3)" }, + }, + optional: {}, + }, "Vector Tile": { required: { urls: { @@ -624,7 +645,7 @@ export async function queryLayerFeatures(layerInfo, map, coordinate, pixel) { features = getVectorTileLayerFeatures(map, pixel); } else if (sourceType === "KML") { features = getKMLLayerFeatures(map, pixel, coordinate, LayerName); - } else if (sourceType === "GeoTIFF") { + } else if (sourceType === "GeoTIFF" || sourceType === "Zarr") { features = getGeoTIFFPixelValues( map, pixel, diff --git a/reactapp/components/modals/DataViewer/DataViewer.js b/reactapp/components/modals/DataViewer/DataViewer.js index e4e9f09b..47c0e5c2 100644 --- a/reactapp/components/modals/DataViewer/DataViewer.js +++ b/reactapp/components/modals/DataViewer/DataViewer.js @@ -254,7 +254,9 @@ function DataViewerModal({ return; } else if ( variableInputValue == null && - !["checkbox", "csv-uploader"].includes(variableInputSource) + !["checkbox", "csv-uploader", "slider"].includes( + variableInputSource, + ) ) { setAlertMessage("Initial value must be selected in the dropdown"); setShowAlert(true); @@ -264,20 +266,25 @@ function DataViewerModal({ vizInputsValues.initial_value = variableInputValue; } - if ( - Object.values(vizInputsValues).every( - (value) => ![null, ""].includes(value), - ) // TODO for csv-uploader, it's ok if data is empty - ) { + const skipInitialValueCheck = ["slider", "csv-uploader"].includes( + vizInputsValues.variable_options_source, + ); + const valuesToValidate = skipInitialValueCheck + ? Object.entries(vizInputsValues) + .filter(([key]) => key !== "initial_value") + .map(([, value]) => value) + : Object.values(vizInputsValues); + if (valuesToValidate.every((value) => ![null, ""].includes(value))) { const { gridItems, id: activeTabId } = getActiveTab(); let updatedGridItems = JSON.parse(JSON.stringify(gridItems)); - updatedGridItems[gridItemIndex].source = vizMetadata.source; + updatedGridItems[gridItemIndex].source = + vizMetadata?.source ?? selectedVizTypeOption.source; updatedGridItems[gridItemIndex].args_string = JSON.stringify( Object.fromEntries( Object.entries(vizInputsValues).map(([key, val]) => [ key, - val.value ?? val, + val?.value ?? val, ]), ), ); diff --git a/reactapp/components/modals/DataViewer/VisualizationPane.js b/reactapp/components/modals/DataViewer/VisualizationPane.js index fb279464..737795dc 100644 --- a/reactapp/components/modals/DataViewer/VisualizationPane.js +++ b/reactapp/components/modals/DataViewer/VisualizationPane.js @@ -278,7 +278,16 @@ function VisualizationPane({ function checkAllInputs() { if (selectedVizTypeOption) { - const allFilled = Object.values(vizInputsValues).every( + // slider's initial value comes from the preview, so don't gate on it + const skipInitialValueCheck = ["slider", "csv-uploader"].includes( + vizInputsValues.variable_options_source, + ); + const inputsToCheck = skipInitialValueCheck + ? Object.entries(vizInputsValues) + .filter(([key]) => key !== "initial_value") + .map(([, value]) => value) + : Object.values(vizInputsValues); + const allFilled = inputsToCheck.every( (value) => !["", null].includes(value), ); const isEmptyArgs = @@ -306,7 +315,7 @@ function VisualizationPane({ args: Object.fromEntries( Object.entries(vizInputsValues).map(([key, val]) => [ key, - val.value ?? val, + val?.value ?? val, ]), ), }; diff --git a/reactapp/components/modals/MapLayer/MapLayer.js b/reactapp/components/modals/MapLayer/MapLayer.js index 34476f1c..d6e60103 100644 --- a/reactapp/components/modals/MapLayer/MapLayer.js +++ b/reactapp/components/modals/MapLayer/MapLayer.js @@ -128,7 +128,7 @@ export function renameLayerInAttributeProps(attributeProps, oldName, newName) { } export const getLayerType = (sourceType) => { - if (sourceType === "GeoTIFF") return "WebGLTile"; + if (sourceType === "GeoTIFF" || sourceType === "Zarr") return "WebGLTile"; if (sourceType.includes("Vector")) return "VectorTileLayer"; if (sourceType.includes("Raster")) return "WebGLTile"; if (sourceType.includes("Tile")) return "TileLayer"; @@ -398,21 +398,26 @@ const MapLayerModal = ({ } } - if (sourceProps.type === "GeoTIFF") { + if (sourceProps.type === "GeoTIFF" || sourceProps.type === "Zarr") { const { rampName, rampMin, rampMax } = sourceProps; - const hasRamp = - typeof rampName === "string" && - rampName.trim() !== "" && + const hasRampName = + typeof rampName === "string" && rampName.trim() !== ""; + const hasRange = typeof rampMin === "string" && rampMin.trim() !== "" && typeof rampMax === "string" && rampMax.trim() !== "" && Number.isFinite(Number(rampMin)) && Number.isFinite(Number(rampMax)); - if (hasRamp) { - const hasNodata = validSourceProps.sources.some( - (s) => s?.nodata !== undefined && s.nodata !== "", - ); + if (hasRampName) { + // Zarr COGs always carry a -9999 nodata sentinel; GeoTIFF depends on + // whether the author set one on any source. + const hasNodata = + sourceProps.type === "Zarr" + ? true + : validSourceProps.sources.some( + (s) => s?.nodata !== undefined && s.nodata !== "", + ); const color = buildGeoTIFFStyleColor({ rampName, rampMin, @@ -421,8 +426,13 @@ const MapLayerModal = ({ }); mapConfiguration.configuration.style = { color }; mapConfiguration.configuration.props.source.rampName = rampName; - mapConfiguration.configuration.props.source.rampMin = rampMin; - mapConfiguration.configuration.props.source.rampMax = rampMax; + // Raw range styles raw band values; auto mode normalizes band 1 from stats. + mapConfiguration.configuration.props.source.props.normalize = !hasRange; + // Persist an explicit range only when set; empty = per-storm auto. + if (hasRange) { + mapConfiguration.configuration.props.source.rampMin = rampMin; + mapConfiguration.configuration.props.source.rampMax = rampMax; + } } } else if (style && style !== "{}") { const apiResponse = await saveLayerJSON({ diff --git a/reactapp/components/modals/MapLayer/RampPicker.js b/reactapp/components/modals/MapLayer/RampPicker.js index ef17a65d..42f2651d 100644 --- a/reactapp/components/modals/MapLayer/RampPicker.js +++ b/reactapp/components/modals/MapLayer/RampPicker.js @@ -32,12 +32,6 @@ const RampRow = styled.button` } `; -const RampLabel = styled.span` - min-width: 90px; - font-size: 0.9rem; - font-weight: 500; -`; - const GradientSwatch = styled.span` flex: 1; height: 20px; @@ -68,7 +62,6 @@ const RampPicker = ({ selectedRamp, onChange }) => { $selected={isSelected} onClick={() => onChange(name)} > - {name}