From 3e74b148e3c4090dd742949ca3eb6c6e35217fa3 Mon Sep 17 00:00:00 2001 From: Jake Gimenes Date: Fri, 21 Aug 2026 15:44:59 -0600 Subject: [PATCH] Add client-side GeoParquet map source type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Render GeoParquet files entirely in the browser via hyparquet + hyparquet-compressors (geoparquet:true decodes WKB geometry to GeoJSON), reprojected with proj4 — no backend conversion endpoint or geopandas. GeoParquet joins the existing client-vector source path, so styling, click-query, and snapping work unchanged. - ModuleLoader.js: loadGeoParquet() reads the file over HTTP(S) via hyparquet, reads the "geo" metadata for the geometry column + CRS, and reprojects features to the map projection. Generalizes the shared registerGeoPackageProjections helper to registerUTMProjections (now used by both GeoPackage and GeoParquet); reuses s3UrlToHttps. - utilities.js: register GeoParquet as a client vector source + config. - Tests for CRS mapping, geo-metadata parsing, and the loader (reprojection, BigInt coercion, empty-geometry, s3, routing). - Docs section for the source. Supersedes the backend GeoParquet implementation (upstream PR #167). Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/source/maps/source_tab.rst | 31 +++ package-lock.json | 30 +++ package.json | 2 + .../components/map/ModuleLoader.test.js | 210 +++++++++++++++++- reactapp/components/map/ModuleLoader.js | 97 +++++++- reactapp/components/map/utilities.js | 7 + 6 files changed, 369 insertions(+), 8 deletions(-) diff --git a/docs/source/maps/source_tab.rst b/docs/source/maps/source_tab.rst index b3005faf..4a0b26f0 100644 --- a/docs/source/maps/source_tab.rst +++ b/docs/source/maps/source_tab.rst @@ -269,6 +269,37 @@ The Zarr source renders a raster layer from a public `Zarr `_ ------------------------------------------------------------------------------------------------------------------------ +++++++++++ +GeoParquet +++++++++++ + +The GeoParquet source renders a **vector** layer from a public `GeoParquet `_ file. TethysDash reads the file entirely in the browser, decodes its geometry, and reprojects it to the map projection — so you supply a file URL rather than a prepared GeoJSON. It is styled and queried like a GeoJSON layer. + +**Layer Properties:** + - **url:** (required) Public URL of the GeoParquet file (an ``https`` URL, or an ``s3://`` URL which is translated to its public ``https`` form). The host must allow cross-origin (CORS) reads. + +.. note:: + The file is downloaded and parsed in the browser, so this suits moderate feature counts; very large files may be slow to render. Files in a projected CRS (e.g. UTM) are reprojected automatically; other custom CRSs may not be supported. GeoParquet's default CRS (OGC:CRS84 / WGS84 lon-lat) is always supported. + +**Example JSON Configuration:** + +:: + + { + "type": "VectorLayer", + "props": { + "name": "Buildings", + "source": { + "type": "GeoParquet", + "props": { + "url": "https://example.com/buildings.parquet" + } + } + } + } + +------------------------------------------------------------------------------------------------------------------------ + ++++++++++ GeoPackage ++++++++++ diff --git a/package-lock.json b/package-lock.json index 73508b7a..18a6856c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,8 @@ "geotiff": "2.1.3", "html-react-parser": "^5.1.18", "html2canvas": "^1.4.1", + "hyparquet": "^1.29.1", + "hyparquet-compressors": "^1.1.1", "json5": "^2.2.3", "ol": "10.4.0", "ol-load-geopackage": "^2.3.1", @@ -11868,6 +11870,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/fzstd": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/fzstd/-/fzstd-0.1.1.tgz", + "integrity": "sha512-dkuVSOKKwh3eas5VkJy1AW1vFpet8TA/fGmVA5krThl8YcOVE/8ZIoEA1+U1vEn5ckxxhLirSdY837azmbaNHA==", + "license": "MIT" + }, "node_modules/generator-function": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/generator-function/-/generator-function-2.0.1.tgz", @@ -13071,6 +13079,28 @@ "node": ">=10.17.0" } }, + "node_modules/hyparquet": { + "version": "1.29.1", + "resolved": "https://registry.npmjs.org/hyparquet/-/hyparquet-1.29.1.tgz", + "integrity": "sha512-Wa57A2KxGFtRTeDTJp4pT0TaqN8btb7/XTrMs17iM+xJxsbBxRyzaGaRRPQ4lOPbmso/qohDafJD7b+cFMZq8A==", + "license": "MIT" + }, + "node_modules/hyparquet-compressors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/hyparquet-compressors/-/hyparquet-compressors-1.1.1.tgz", + "integrity": "sha512-yx7aA3Rhj0YycbdV71+XznQSLAefa4cT0urpgNXy4aM6eSeCknaVDNne8y45Uz74Fb15yyXUzOStlceOJBan7A==", + "license": "MIT", + "dependencies": { + "fzstd": "0.1.1", + "hysnappy": "1.0.0" + } + }, + "node_modules/hysnappy": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/hysnappy/-/hysnappy-1.0.0.tgz", + "integrity": "sha512-MNrC4NfwDGPb889O6gIfEtbvEZCSWUsSEhsz4Oq2FRcpGtXHfeVz3KciSPp5Pnnz1NjFMgDQNfxdJozymJEDDA==", + "license": "MIT" + }, "node_modules/iconv-lite": { "version": "0.7.2", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", diff --git a/package.json b/package.json index ab95caf3..773f5ff1 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,8 @@ "geotiff": "2.1.3", "html-react-parser": "^5.1.18", "html2canvas": "^1.4.1", + "hyparquet": "^1.29.1", + "hyparquet-compressors": "^1.1.1", "json5": "^2.2.3", "ol": "10.4.0", "ol-load-geopackage": "^2.3.1", diff --git a/reactapp/__tests__/components/map/ModuleLoader.test.js b/reactapp/__tests__/components/map/ModuleLoader.test.js index 1dc69435..3e2d1ab6 100644 --- a/reactapp/__tests__/components/map/ModuleLoader.test.js +++ b/reactapp/__tests__/components/map/ModuleLoader.test.js @@ -15,8 +15,12 @@ import moduleLoader, { applyAutoRamp, loadGeoPackage, s3UrlToHttps, - registerGeoPackageProjections, + registerUTMProjections, GeoPackageError, + loadGeoParquet, + geoParquetCRSToProjection, + readGeoParquetGeoMetadata, + GeoParquetError, } from "components/map/ModuleLoader"; import { fromUrl } from "geotiff"; import WebGLTile from "ol/layer/WebGLTile.js"; @@ -57,6 +61,11 @@ import { import { get as getProjection } from "ol/proj"; import proj4 from "proj4"; import { loadGpkg } from "ol-load-geopackage"; +import { + asyncBufferFromUrl, + parquetMetadataAsync, + parquetReadObjects, +} from "hyparquet"; jest.mock("geotiff", () => ({ fromUrl: jest.fn() })); @@ -66,6 +75,20 @@ jest.mock("ol-load-geopackage", () => ({ loadGpkg: jest.fn(), })); +jest.mock( + "hyparquet", + () => ({ + __esModule: true, + asyncBufferFromUrl: jest.fn(), + parquetMetadataAsync: jest.fn(), + parquetReadObjects: jest.fn(), + }), + { virtual: true }, +); +jest.mock("hyparquet-compressors", () => ({ compressors: { __mock: true } }), { + virtual: true, +}); + jest.mock("ol/source/GeoTIFF.js", () => { const ActualSource = jest.requireActual("ol/source/Source.js").default; const spy = jest.fn(); @@ -2631,9 +2654,9 @@ describe("s3UrlToHttps", () => { }); }); -describe("registerGeoPackageProjections", () => { +describe("registerUTMProjections", () => { test("registers WGS84 UTM zones with proj4 and OpenLayers", () => { - registerGeoPackageProjections(); + registerUTMProjections(); expect(proj4.defs("EPSG:32615")).toBeTruthy(); expect(proj4.defs("EPSG:32715")).toBeTruthy(); expect(getProjection("EPSG:32615")).not.toBeNull(); @@ -2641,8 +2664,8 @@ describe("registerGeoPackageProjections", () => { test("is idempotent", () => { expect(() => { - registerGeoPackageProjections(); - registerGeoPackageProjections(); + registerUTMProjections(); + registerUTMProjections(); }).not.toThrow(); }); }); @@ -2737,3 +2760,180 @@ describe("loadGeoPackage", () => { expect(out).toBe(src); }); }); + +describe("geoParquetCRSToProjection", () => { + test("treats null/undefined CRS as WGS84 (spec default)", () => { + expect(geoParquetCRSToProjection(null)).toBe("EPSG:4326"); + expect(geoParquetCRSToProjection(undefined)).toBe("EPSG:4326"); + }); + + test("resolves a PROJJSON id to its projection code", () => { + expect( + geoParquetCRSToProjection({ id: { authority: "EPSG", code: 3857 } }), + ).toBe("EPSG:3857"); + }); + + test("falls back to the first entry of an ids array", () => { + expect( + geoParquetCRSToProjection({ ids: [{ authority: "EPSG", code: 32615 }] }), + ).toBe("EPSG:32615"); + }); +}); + +describe("readGeoParquetGeoMetadata", () => { + test("defaults to 'geometry' + WGS84 when there is no geo key", () => { + expect(readGeoParquetGeoMetadata({})).toEqual({ + geometryColumn: "geometry", + dataProjection: "EPSG:4326", + }); + }); + + test("reads primary_column and its CRS from the geo metadata", () => { + const geo = JSON.stringify({ + primary_column: "geom", + columns: { + geom: { + encoding: "WKB", + crs: { id: { authority: "EPSG", code: 32615 } }, + }, + }, + }); + const meta = { key_value_metadata: [{ key: "geo", value: geo }] }; + expect(readGeoParquetGeoMetadata(meta)).toEqual({ + geometryColumn: "geom", + dataProjection: "EPSG:32615", + }); + }); +}); + +describe("loadGeoParquet", () => { + const pointRows = [ + { geometry: { type: "Point", coordinates: [-100, 40] }, name: "A" }, + { geometry: { type: "Point", coordinates: [-90, 35] }, name: "B" }, + ]; + const noGeoMeta = { key_value_metadata: undefined }; + + test("reads a GeoParquet file into an OL VectorSource of features", async () => { + asyncBufferFromUrl.mockResolvedValue({}); + parquetMetadataAsync.mockResolvedValue(noGeoMeta); + parquetReadObjects.mockResolvedValue(pointRows); + + const source = await loadGeoParquet( + { type: "GeoParquet", props: { url: "https://x/data.parquet" } }, + "EPSG:3857", + ); + expect(source).toBeInstanceOf(VectorSource); + const features = source.getFeatures(); + expect(features).toHaveLength(2); + expect(features[0].get("name")).toBe("A"); + expect(features[0].getGeometry().getType()).toBe("Point"); + // -100 lon in EPSG:3857 is a large negative metre value: reprojection ran. + expect(features[0].getGeometry().getCoordinates()[0]).toBeCloseTo( + -11131949.08, + 0, + ); + }); + + test("passes the extended codecs and geoparquet flag to the reader", async () => { + asyncBufferFromUrl.mockResolvedValue({}); + parquetMetadataAsync.mockResolvedValue(noGeoMeta); + parquetReadObjects.mockResolvedValue([]); + + await loadGeoParquet( + { type: "GeoParquet", props: { url: "https://x/data.parquet" } }, + "EPSG:3857", + ); + expect(parquetReadObjects).toHaveBeenCalledWith( + expect.objectContaining({ + compressors: { __mock: true }, + geoparquet: true, + }), + ); + }); + + test("coerces BigInt property values to Number", async () => { + asyncBufferFromUrl.mockResolvedValue({}); + parquetMetadataAsync.mockResolvedValue(noGeoMeta); + parquetReadObjects.mockResolvedValue([ + { geometry: { type: "Point", coordinates: [0, 0] }, id: 42n }, + ]); + const source = await loadGeoParquet( + { type: "GeoParquet", props: { url: "https://x/d.parquet" } }, + "EPSG:3857", + ); + expect(source.getFeatures()[0].get("id")).toBe(42); + }); + + test("drops rows with no geometry", async () => { + asyncBufferFromUrl.mockResolvedValue({}); + parquetMetadataAsync.mockResolvedValue(noGeoMeta); + parquetReadObjects.mockResolvedValue([ + { geometry: { type: "Point", coordinates: [0, 0] }, name: "keep" }, + { geometry: null, name: "drop" }, + ]); + const source = await loadGeoParquet( + { type: "GeoParquet", props: { url: "https://x/d.parquet" } }, + "EPSG:3857", + ); + expect(source.getFeatures()).toHaveLength(1); + expect(source.getFeatures()[0].get("name")).toBe("keep"); + }); + + test("uses the primary geometry column from geo metadata", async () => { + asyncBufferFromUrl.mockResolvedValue({}); + parquetMetadataAsync.mockResolvedValue({ + key_value_metadata: [ + { + key: "geo", + value: JSON.stringify({ + primary_column: "geom", + columns: { geom: { encoding: "WKB", crs: null } }, + }), + }, + ], + }); + parquetReadObjects.mockResolvedValue([ + { geom: { type: "Point", coordinates: [0, 0] }, name: "A" }, + ]); + const source = await loadGeoParquet( + { type: "GeoParquet", props: { url: "https://x/d.parquet" } }, + "EPSG:3857", + ); + expect(source.getFeatures()).toHaveLength(1); + expect(source.getFeatures()[0].getGeometry().getType()).toBe("Point"); + }); + + test("translates an s3:// url before fetching", async () => { + asyncBufferFromUrl.mockResolvedValue({}); + parquetMetadataAsync.mockResolvedValue(noGeoMeta); + parquetReadObjects.mockResolvedValue([]); + await loadGeoParquet( + { type: "GeoParquet", props: { url: "s3://b-us-east-1-x/d.parquet" } }, + "EPSG:3857", + ); + expect(asyncBufferFromUrl).toHaveBeenCalledWith({ + url: "https://b-us-east-1-x.s3.us-east-1.amazonaws.com/d.parquet", + }); + }); + + test("rejects a missing url without touching the network", async () => { + await expect( + loadGeoParquet({ type: "GeoParquet", props: {} }, "EPSG:3857"), + ).rejects.toThrow(GeoParquetError); + expect(asyncBufferFromUrl).not.toHaveBeenCalled(); + }); + + test("moduleLoader routes a GeoParquet config to loadGeoParquet", async () => { + asyncBufferFromUrl.mockResolvedValue({}); + parquetMetadataAsync.mockResolvedValue(noGeoMeta); + parquetReadObjects.mockResolvedValue([ + { geometry: { type: "Point", coordinates: [0, 0] }, name: "A" }, + ]); + const source = await moduleLoader( + { type: "GeoParquet", props: { url: "https://x/d.parquet" } }, + "EPSG:3857", + ); + expect(source).toBeInstanceOf(VectorSource); + expect(source.getFeatures()).toHaveLength(1); + }); +}); diff --git a/reactapp/components/map/ModuleLoader.js b/reactapp/components/map/ModuleLoader.js index ee9c6b57..0679eeb6 100644 --- a/reactapp/components/map/ModuleLoader.js +++ b/reactapp/components/map/ModuleLoader.js @@ -330,9 +330,9 @@ export function s3UrlToHttps(url, defaultRegion = "us-east-1") { return `https://${bucket}.s3.${region}.amazonaws.com/${key}`; } -// Register proj4 + all WGS84 UTM zones so ol can reproject projected GeoPackages. +// Register proj4 + all WGS84 UTM zones so ol can reproject projected sources. let projectionsRegistered = false; -export function registerGeoPackageProjections() { +export function registerUTMProjections() { if (projectionsRegistered) return; for (let zone = 1; zone <= 60; zone++) { proj4.defs( @@ -374,7 +374,7 @@ export async function loadGeoPackage(config, mapProjection) { } const url = s3UrlToHttps(rawUrl); - registerGeoPackageProjections(); + registerUTMProjections(); const { loadGpkg } = await getGeoPackageLib(); const cacheKey = `${url}::${mapProjection}`; @@ -399,10 +399,101 @@ export async function loadGeoPackage(config, mapProjection) { return source; } +export class GeoParquetError extends Error {} + +// Map a GeoParquet column CRS (PROJJSON) to an OL projection code. A null/absent +// CRS means OGC:CRS84 (lon/lat WGS84) per the GeoParquet spec. +export function geoParquetCRSToProjection(crs) { + if (crs === null || crs === undefined) return "EPSG:4326"; + const id = crs.id ?? crs.ids?.[0]; + return id ? `${id.authority}:${id.code}` : "EPSG:4326"; +} + +// Read the GeoParquet "geo" file metadata: primary geometry column + its CRS. +export function readGeoParquetGeoMetadata(metadata) { + const geoValue = metadata?.key_value_metadata?.find( + (kv) => kv.key === "geo", + )?.value; + if (!geoValue) { + return { geometryColumn: "geometry", dataProjection: "EPSG:4326" }; + } + const geo = JSON.parse(geoValue); + const geometryColumn = geo.primary_column || "geometry"; + const dataProjection = geoParquetCRSToProjection( + geo.columns?.[geometryColumn]?.crs, + ); + return { geometryColumn, dataProjection }; +} + +// Lazy-load hyparquet + its codec pack; only needed when a GeoParquet renders. +let hyparquetLib = null; +async function getHyparquet() { + if (!hyparquetLib) { + const [hp, comp] = await Promise.all([ + import("hyparquet"), + import("hyparquet-compressors"), + ]); + hyparquetLib = { ...hp, compressors: comp.compressors }; + } + return hyparquetLib; +} + +// Read a GeoParquet file in-browser as a reprojected OL vector source. hyparquet +// decodes the WKB geometry column to GeoJSON (geoparquet:true); features are then +// reprojected from the file's declared CRS to the map projection. +export async function loadGeoParquet(config, mapProjection) { + const rawUrl = config.props?.url; + if (!rawUrl) { + throw new GeoParquetError("GeoParquet source requires a file URL"); + } + const url = s3UrlToHttps(rawUrl); + registerUTMProjections(); + + const { + asyncBufferFromUrl, + parquetMetadataAsync, + parquetReadObjects, + compressors, + } = await getHyparquet(); + + const file = await asyncBufferFromUrl({ url }); + const metadata = await parquetMetadataAsync(file); + const { geometryColumn, dataProjection } = + readGeoParquetGeoMetadata(metadata); + + const rows = await parquetReadObjects({ + file, + compressors, + geoparquet: true, + }); + const features = rows + .map((row) => { + const { [geometryColumn]: geometry, ...rest } = row; + // Parquet INT64 columns arrive as BigInt, which breaks JSON serialization + // in the popup/click path, so coerce them to Number. + const properties = {}; + for (const [key, value] of Object.entries(rest)) { + properties[key] = typeof value === "bigint" ? Number(value) : value; + } + return { type: "Feature", geometry: geometry ?? null, properties }; + }) + .filter((feature) => feature.geometry != null); + + return new VectorSource({ + features: new GeoJSON().readFeatures( + { type: "FeatureCollection", features }, + { dataProjection, featureProjection: mapProjection }, + ), + }); +} + const moduleLoader = async (config, mapProjection) => { if (config.type === "GeoPackage") { return loadGeoPackage(config, mapProjection); } + if (config.type === "GeoParquet") { + return loadGeoParquet(config, mapProjection); + } if (config.type === "Zarr") { // Already yields OL's `sources` shape, so it skips the GeoTIFF branch below. config = zarrSourceToGeoTIFF(config); diff --git a/reactapp/components/map/utilities.js b/reactapp/components/map/utilities.js index 532b02cb..2b4b6af7 100644 --- a/reactapp/components/map/utilities.js +++ b/reactapp/components/map/utilities.js @@ -22,6 +22,7 @@ export const CLIENT_VECTOR_SOURCE_TYPES = [ "GeoJSON", "ESRI Feature Service", "GeoPackage", + "GeoParquet", ]; // Coerce an optional numeric layer prop: GUI inputs emit strings, so accept @@ -146,6 +147,12 @@ export const sourcePropertiesOptions = { mask_below: { placeholder: "Mask values at or below this" }, }, }, + GeoParquet: { + required: { + url: { placeholder: "GeoParquet file URL (https or s3)" }, + }, + optional: {}, + }, GeoPackage: { required: { url: { placeholder: "GeoPackage file URL (https or s3)" },