From e1a6e97f48f505b56ad8b5a0b2c76c0a51503f51 Mon Sep 17 00:00:00 2001 From: Jake Gimenes Date: Fri, 21 Aug 2026 10:45:15 -0600 Subject: [PATCH 1/2] Add client-side GeoPackage map source type Render GeoPackage (.gpkg) vector tables entirely in the browser via ol-load-geopackage + sql.js (WebAssembly SQLite), reprojected with proj4, with no backend conversion endpoint. GeoPackage joins the existing client-vector source path so styling, click-query, and snapping work unchanged; one map layer maps to one feature table (url + layer name). - ModuleLoader.js: loadGeoPackage() with s3://->https translation, WGS84 UTM proj4 registration, lazy lib/wasm init, and parse-once cache keyed by url+projection (one file often backs several layers). - utilities.js: register GeoPackage as a client vector source + config. - webpack.config.js: emit sql-wasm.wasm; stub node-core modules for browser. - Tests for URL translation, CRS registration, table select, caching, and error paths; docs section for the source. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/source/maps/source_tab.rst | 33 ++++ package-lock.json | 55 ++++++ package.json | 5 +- .../components/map/ModuleLoader.test.js | 156 ++++++++++++++++++ reactapp/components/map/ModuleLoader.js | 89 ++++++++++ reactapp/components/map/utilities.js | 13 +- reactapp/config/webpack.config.js | 8 + 7 files changed, 357 insertions(+), 2 deletions(-) diff --git a/docs/source/maps/source_tab.rst b/docs/source/maps/source_tab.rst index 78ec60e5..b3005faf 100644 --- a/docs/source/maps/source_tab.rst +++ b/docs/source/maps/source_tab.rst @@ -269,6 +269,39 @@ The Zarr source renders a raster layer from a public `Zarr `_ ------------------------------------------------------------------------------------------------------------------------ +++++++++++ +GeoPackage +++++++++++ + +The GeoPackage source renders **vector** layers from a public `GeoPackage `_ (``.gpkg``) file. A GeoPackage is an OGC SQLite database that may hold several feature tables, so you supply a file URL **and** the table (layer) name — one map layer per table. TethysDash reads the file entirely in the browser and reprojects it to the map projection, so no server-side conversion is involved. Layers are styled and queried like a GeoJSON layer. + +**Layer Properties:** + - **url:** (required) Public URL of the GeoPackage file (an ``https`` URL, or an ``s3://`` URL which is translated to its public ``https`` form). The host must allow cross-origin (CORS) reads. + - **layer:** (required) Name of the feature table within the GeoPackage to render. + +.. note:: + The file is downloaded and parsed in the browser, so this suits moderate feature counts; very large tables may be slow to render. Feature tables in projected CRSs (e.g. UTM) are reprojected automatically; other custom CRSs may not be supported. + +**Example JSON Configuration:** + +:: + + { + "type": "VectorLayer", + "props": { + "name": "Buildings", + "source": { + "type": "GeoPackage", + "props": { + "url": "https://example.com/flood_event.gpkg", + "layer": "buildings" + } + } + } + } + +------------------------------------------------------------------------------------------------------------------------ + +++++++++++++ Custom Layers +++++++++++++ diff --git a/package-lock.json b/package-lock.json index 28f92993..73508b7a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,10 +38,12 @@ "html2canvas": "^1.4.1", "json5": "^2.2.3", "ol": "10.4.0", + "ol-load-geopackage": "^2.3.1", "ol-mapbox-style": "12.4.0", "ol-pmtiles": "^2.0.2", "plotly.js-strict-dist-min": "^2.35.2", "prismjs": "^1.28.0", + "proj4": "^2.21.0", "rc-slider": "^11.1.8", "react": "^18.3.1", "react-bootstrap": "^2.10.2", @@ -63,6 +65,7 @@ "sass": "^1.49.0", "sass-loader": "^12.3.0", "simple-xml-to-json": "^1.2.3", + "sql.js": "1.13.0", "style-loader": "^3.3.1", "styled-components": "^6.3.9", "swiper": "^11.2.1", @@ -16809,6 +16812,12 @@ "node": ">= 0.6" } }, + "node_modules/mgrs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mgrs/-/mgrs-1.0.0.tgz", + "integrity": "sha512-awNbTOqCxK1DBGjalK3xqWIstBZgN6fxsMSiXLs9/spqWkF2pAhb2rrYCFSsr1/tT7PhcDGjZndG8SWYn0byYA==", + "license": "MIT" + }, "node_modules/micromatch": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", @@ -17474,6 +17483,16 @@ "url": "https://opencollective.com/openlayers" } }, + "node_modules/ol-load-geopackage": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/ol-load-geopackage/-/ol-load-geopackage-2.3.1.tgz", + "integrity": "sha512-ijyxZ/09j7a4fTTZOSg7gzm2u0p1oBIOCZY7PB3G8WDb49WkRTc9Sk1ewGiBNs/68Qavb1jFEjrm9ZloHmddAg==", + "license": "ISC", + "dependencies": { + "ol": ">=6.7.0", + "sql.js": "1.13.0" + } + }, "node_modules/ol-mapbox-style": { "version": "12.4.0", "resolved": "https://registry.npmjs.org/ol-mapbox-style/-/ol-mapbox-style-12.4.0.tgz", @@ -18380,6 +18399,27 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "license": "MIT" }, + "node_modules/proj4": { + "version": "2.21.0", + "resolved": "https://registry.npmjs.org/proj4/-/proj4-2.21.0.tgz", + "integrity": "sha512-33HfDftqw8kY+Cl1dcL16SJuqTSzYxmz4re7Nmhk+fs1/N1fFrAkkF579msQTR/4fLeJVSNne8/gpoCz0fk1uw==", + "license": "MIT", + "dependencies": { + "mgrs": "1.0.0", + "wkt-parser": "^1.5.5" + }, + "funding": { + "url": "https://github.com/sponsors/ahocevar" + }, + "peerDependencies": { + "geotiff": "*" + }, + "peerDependenciesMeta": { + "geotiff": { + "optional": true + } + } + }, "node_modules/promise": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/promise/-/promise-8.3.0.tgz", @@ -20670,6 +20710,12 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/sql.js": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/sql.js/-/sql.js-1.13.0.tgz", + "integrity": "sha512-RJbVP1HRDlUUXahJ7VMTcu9Rm1Nzw+EBpoPr94vnbD4LwR715F3CcxE2G2k45PewcaZ57pjetYa+LoSJLAASgA==", + "license": "MIT" + }, "node_modules/stack-trace": { "version": "0.0.9", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz", @@ -23155,6 +23201,15 @@ "dev": true, "license": "MIT" }, + "node_modules/wkt-parser": { + "version": "1.5.6", + "resolved": "https://registry.npmjs.org/wkt-parser/-/wkt-parser-1.5.6.tgz", + "integrity": "sha512-cqHU3lzGt/gt2OqIORP0uVy5yeOX43WABmgFkmJsmcqH3HhQo9PiJG6ftytwGKmpQUbegeX7+pQc2BuNCRfcrw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ahocevar" + } + }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", diff --git a/package.json b/package.json index 198f2531..ab95caf3 100644 --- a/package.json +++ b/package.json @@ -48,10 +48,12 @@ "html2canvas": "^1.4.1", "json5": "^2.2.3", "ol": "10.4.0", + "ol-load-geopackage": "^2.3.1", "ol-mapbox-style": "12.4.0", "ol-pmtiles": "^2.0.2", "plotly.js-strict-dist-min": "^2.35.2", "prismjs": "^1.28.0", + "proj4": "^2.21.0", "rc-slider": "^11.1.8", "react": "^18.3.1", "react-bootstrap": "^2.10.2", @@ -73,6 +75,7 @@ "sass": "^1.49.0", "sass-loader": "^12.3.0", "simple-xml-to-json": "^1.2.3", + "sql.js": "1.13.0", "style-loader": "^3.3.1", "styled-components": "^6.3.9", "swiper": "^11.2.1", @@ -219,7 +222,7 @@ "/node_modules" ], "moduleNameMapper": { - "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "/reactapp/__tests__/utilities/fileMock.js", + "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga|wasm)$": "/reactapp/__tests__/utilities/fileMock.js", "^.+\\.module\\.(css|sass|scss|less)$": "identity-obj-proxy", "\\.(css|less|scss|sss|styl)$": "/node_modules/jest-css-modules" }, diff --git a/reactapp/__tests__/components/map/ModuleLoader.test.js b/reactapp/__tests__/components/map/ModuleLoader.test.js index 5ecbd821..1dc69435 100644 --- a/reactapp/__tests__/components/map/ModuleLoader.test.js +++ b/reactapp/__tests__/components/map/ModuleLoader.test.js @@ -13,6 +13,10 @@ import moduleLoader, { withAntimeridianFix, zarrSourceToGeoTIFF, applyAutoRamp, + loadGeoPackage, + s3UrlToHttps, + registerGeoPackageProjections, + GeoPackageError, } from "components/map/ModuleLoader"; import { fromUrl } from "geotiff"; import WebGLTile from "ol/layer/WebGLTile.js"; @@ -50,9 +54,18 @@ import { defaultStroke, defaultStrokeWidth, } from "components/inputs/RuleEditor.js"; +import { get as getProjection } from "ol/proj"; +import proj4 from "proj4"; +import { loadGpkg } from "ol-load-geopackage"; jest.mock("geotiff", () => ({ fromUrl: jest.fn() })); +jest.mock("ol-load-geopackage", () => ({ + __esModule: true, + initSqlJsWasm: jest.fn(), + loadGpkg: jest.fn(), +})); + jest.mock("ol/source/GeoTIFF.js", () => { const ActualSource = jest.requireActual("ol/source/Source.js").default; const spy = jest.fn(); @@ -2581,3 +2594,146 @@ describe("applyAutoRamp", () => { }); }); }); + +describe("s3UrlToHttps", () => { + test("passes an https url through unchanged", () => { + expect(s3UrlToHttps("https://x/y.gpkg")).toBe("https://x/y.gpkg"); + }); + + test("returns non-string input unchanged", () => { + expect(s3UrlToHttps(undefined)).toBeUndefined(); + }); + + test("converts s3:// to a virtual-hosted https url, sniffing the region", () => { + expect( + s3UrlToHttps("s3://cog-s3-test-1234-us-east-1-an/Guatemala_IBF/f.gpkg"), + ).toBe( + "https://cog-s3-test-1234-us-east-1-an.s3.us-east-1.amazonaws.com/Guatemala_IBF/f.gpkg", + ); + }); + + test("honors a region embedded in the bucket name", () => { + expect(s3UrlToHttps("s3://data-eu-west-2-x/k.gpkg")).toBe( + "https://data-eu-west-2-x.s3.eu-west-2.amazonaws.com/k.gpkg", + ); + }); + + test("defaults the region when none is in the bucket name", () => { + expect(s3UrlToHttps("s3://my-bucket/a/b.gpkg")).toBe( + "https://my-bucket.s3.us-east-1.amazonaws.com/a/b.gpkg", + ); + }); + + test("handles an s3 url with no key", () => { + expect(s3UrlToHttps("s3://plain-bucket")).toBe( + "https://plain-bucket.s3.us-east-1.amazonaws.com/", + ); + }); +}); + +describe("registerGeoPackageProjections", () => { + test("registers WGS84 UTM zones with proj4 and OpenLayers", () => { + registerGeoPackageProjections(); + expect(proj4.defs("EPSG:32615")).toBeTruthy(); + expect(proj4.defs("EPSG:32715")).toBeTruthy(); + expect(getProjection("EPSG:32615")).not.toBeNull(); + }); + + test("is idempotent", () => { + expect(() => { + registerGeoPackageProjections(); + registerGeoPackageProjections(); + }).not.toThrow(); + }); +}); + +describe("loadGeoPackage", () => { + test("returns the requested table's vector source", async () => { + const src = new VectorSource(); + loadGpkg.mockResolvedValue([{ roads: src }, {}]); + const out = await loadGeoPackage( + { props: { url: "https://h/t1.gpkg", layer: "roads" } }, + "EPSG:3857", + ); + expect(out).toBe(src); + expect(loadGpkg).toHaveBeenCalledWith("https://h/t1.gpkg", "EPSG:3857"); + }); + + test("translates an s3:// url before loading", async () => { + loadGpkg.mockResolvedValue([{ t: new VectorSource() }, {}]); + await loadGeoPackage( + { props: { url: "s3://b-us-east-1-x/t2.gpkg", layer: "t" } }, + "EPSG:3857", + ); + expect(loadGpkg).toHaveBeenCalledWith( + "https://b-us-east-1-x.s3.us-east-1.amazonaws.com/t2.gpkg", + "EPSG:3857", + ); + }); + + test("rejects when the url is missing", async () => { + await expect( + loadGeoPackage({ props: { layer: "t" } }, "EPSG:3857"), + ).rejects.toThrow(GeoPackageError); + expect(loadGpkg).not.toHaveBeenCalled(); + }); + + test("rejects when the table name is missing", async () => { + await expect( + loadGeoPackage({ props: { url: "https://h/t3.gpkg" } }, "EPSG:3857"), + ).rejects.toThrow(GeoPackageError); + }); + + test("rejects an unknown table, listing the available ones", async () => { + loadGpkg.mockResolvedValue([ + { roads: new VectorSource(), bldgs: new VectorSource() }, + {}, + ]); + await expect( + loadGeoPackage( + { props: { url: "https://h/t4.gpkg", layer: "nope" } }, + "EPSG:3857", + ), + ).rejects.toThrow(/roads|bldgs/); + }); + + test("parses a file only once across layers (cache by url+projection)", async () => { + loadGpkg.mockResolvedValue([{ t: new VectorSource() }, {}]); + const cfg = { props: { url: "https://h/t5.gpkg", layer: "t" } }; + await loadGeoPackage(cfg, "EPSG:3857"); + await loadGeoPackage(cfg, "EPSG:3857"); + expect(loadGpkg).toHaveBeenCalledTimes(1); + }); + + test("does not cache a failed load", async () => { + loadGpkg + .mockRejectedValueOnce(new Error("boom")) + .mockResolvedValue([{ t: new VectorSource() }, {}]); + const cfg = { props: { url: "https://h/t6.gpkg", layer: "t" } }; + await expect(loadGeoPackage(cfg, "EPSG:3857")).rejects.toThrow("boom"); + const out = await loadGeoPackage(cfg, "EPSG:3857"); + expect(out).toBeInstanceOf(VectorSource); + expect(loadGpkg).toHaveBeenCalledTimes(2); + }); + + test("caches separately per display projection", async () => { + loadGpkg.mockResolvedValue([{ t: new VectorSource() }, {}]); + const cfg = { props: { url: "https://h/t7.gpkg", layer: "t" } }; + await loadGeoPackage(cfg, "EPSG:3857"); + await loadGeoPackage(cfg, "EPSG:4326"); + expect(loadGpkg).toHaveBeenCalledTimes(2); + }); + + test("moduleLoader routes a GeoPackage config to loadGeoPackage", async () => { + const src = new VectorSource(); + loadGpkg.mockResolvedValue([{ roads: src }, {}]); + const out = await moduleLoader( + { + type: "GeoPackage", + props: { url: "https://h/t8.gpkg", layer: "roads" }, + }, + "EPSG:3857", + ); + expect(out).toBe(src); + }); +}); diff --git a/reactapp/components/map/ModuleLoader.js b/reactapp/components/map/ModuleLoader.js index b34c5c14..ee9c6b57 100644 --- a/reactapp/components/map/ModuleLoader.js +++ b/reactapp/components/map/ModuleLoader.js @@ -32,6 +32,9 @@ import { buildCategoricalStyleColor, isUsableClass, } from "components/map/geoTIFFStyle"; +import proj4 from "proj4"; +import { register as registerProj4 } from "ol/proj/proj4.js"; +import sqlWasmUrl from "sql.js/dist/sql-wasm.wasm"; const moduleCache = {}; const styleCache = new Map(); @@ -313,7 +316,93 @@ export async function applyAutoRamp(layerConfig) { return layerConfig; } +export class GeoPackageError extends Error {} + +// s3://bucket/key -> virtual-hosted https so the browser can fetch it directly. +export function s3UrlToHttps(url, defaultRegion = "us-east-1") { + if (typeof url !== "string" || !url.startsWith("s3://")) return url; + const rest = url.slice(5); + const slash = rest.indexOf("/"); + const bucket = slash === -1 ? rest : rest.slice(0, slash); + const key = slash === -1 ? "" : rest.slice(slash + 1); + const region = + bucket.match(/(us|eu|ap|sa|ca|me|af)-[a-z]+-\d+/)?.[0] ?? defaultRegion; + return `https://${bucket}.s3.${region}.amazonaws.com/${key}`; +} + +// Register proj4 + all WGS84 UTM zones so ol can reproject projected GeoPackages. +let projectionsRegistered = false; +export function registerGeoPackageProjections() { + if (projectionsRegistered) return; + for (let zone = 1; zone <= 60; zone++) { + proj4.defs( + `EPSG:${32600 + zone}`, + `+proj=utm +zone=${zone} +datum=WGS84 +units=m +no_defs +type=crs`, + ); + proj4.defs( + `EPSG:${32700 + zone}`, + `+proj=utm +zone=${zone} +south +datum=WGS84 +units=m +no_defs +type=crs`, + ); + } + registerProj4(proj4); + projectionsRegistered = true; +} + +// Lazy-load ol-load-geopackage and init the sql.js wasm loader once. +let geoPackageLib = null; +async function getGeoPackageLib() { + if (!geoPackageLib) { + const lib = await import("ol-load-geopackage"); + lib.initSqlJsWasm(sqlWasmUrl.replace(/\/sql-wasm\.wasm$/, "")); + geoPackageLib = lib; + } + return geoPackageLib; +} + +// Cache parsed gpkg per url+projection; one file often backs several layers. +const geoPackageCache = new Map(); + +// Read one GeoPackage table in-browser as a reprojected OL vector source. +export async function loadGeoPackage(config, mapProjection) { + const rawUrl = config.props?.url; + const table = config.props?.layer; + if (!rawUrl) { + throw new GeoPackageError("GeoPackage source requires a file URL"); + } + if (!table) { + throw new GeoPackageError("GeoPackage source requires a table name"); + } + + const url = s3UrlToHttps(rawUrl); + registerGeoPackageProjections(); + const { loadGpkg } = await getGeoPackageLib(); + + const cacheKey = `${url}::${mapProjection}`; + if (!geoPackageCache.has(cacheKey)) { + geoPackageCache.set(cacheKey, loadGpkg(url, mapProjection)); + } + let dataByTable; + try { + [dataByTable] = await geoPackageCache.get(cacheKey); + } catch (error) { + geoPackageCache.delete(cacheKey); + throw error; + } + + const source = dataByTable[table]; + if (!source) { + throw new GeoPackageError( + `Table "${table}" not found in GeoPackage. Available tables: ` + + Object.keys(dataByTable).join(", "), + ); + } + return source; +} + const moduleLoader = async (config, mapProjection) => { + if (config.type === "GeoPackage") { + return loadGeoPackage(config, mapProjection); + } if (config.type === "Zarr") { // Already yields OL's `sources` shape, so it skips the GeoTIFF branch below. config = zarrSourceToGeoTIFF(config); diff --git a/reactapp/components/map/utilities.js b/reactapp/components/map/utilities.js index ef331413..532b02cb 100644 --- a/reactapp/components/map/utilities.js +++ b/reactapp/components/map/utilities.js @@ -18,7 +18,11 @@ import Protobuf from "pbf"; // Source types whose features live in a client-side OL VectorSource (vs // server-rendered services queried remotely). -export const CLIENT_VECTOR_SOURCE_TYPES = ["GeoJSON", "ESRI Feature Service"]; +export const CLIENT_VECTOR_SOURCE_TYPES = [ + "GeoJSON", + "ESRI Feature Service", + "GeoPackage", +]; // Coerce an optional numeric layer prop: GUI inputs emit strings, so accept // any numeric value but treat null/undefined/blank/non-numeric as unset. @@ -142,6 +146,13 @@ export const sourcePropertiesOptions = { mask_below: { placeholder: "Mask values at or below this" }, }, }, + GeoPackage: { + required: { + url: { placeholder: "GeoPackage file URL (https or s3)" }, + layer: { placeholder: "Table (layer) name" }, + }, + optional: {}, + }, "Vector Tile": { required: { urls: { diff --git a/reactapp/config/webpack.config.js b/reactapp/config/webpack.config.js index c0a52c83..6a2bbf8f 100644 --- a/reactapp/config/webpack.config.js +++ b/reactapp/config/webpack.config.js @@ -35,6 +35,8 @@ module.exports = (env, argv) => { path.resolve(__dirname, "../"), path.resolve(__dirname, "../../node_modules"), ], + // sql.js references node core modules in its node path; stub for browser. + fallback: { fs: false, path: false, crypto: false }, }, plugins: [ new Dotenv({ @@ -110,6 +112,12 @@ module.exports = (env, argv) => { }, ], }, + { + // Emit sql-wasm.wasm verbatim so ol-load-geopackage can fetch it. + test: /sql-wasm\.wasm$/, + type: "asset/resource", + generator: { filename: "sql-wasm.wasm" }, + }, ], }, optimization: { From 164fb8d3a74cd1b13ef0957c3a2b09d7a3e33b7e Mon Sep 17 00:00:00 2001 From: Jake Gimenes Date: Wed, 2 Sep 2026 12:54:12 -0600 Subject: [PATCH 2/2] Fix ModuleLoader.test.js braces dropped in a main merge --- reactapp/__tests__/components/map/ModuleLoader.test.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/reactapp/__tests__/components/map/ModuleLoader.test.js b/reactapp/__tests__/components/map/ModuleLoader.test.js index ab177ba1..1dc1051c 100644 --- a/reactapp/__tests__/components/map/ModuleLoader.test.js +++ b/reactapp/__tests__/components/map/ModuleLoader.test.js @@ -2735,6 +2735,9 @@ describe("loadGeoPackage", () => { "EPSG:3857", ); expect(out).toBe(src); + }); +}); + describe("matchesCondition — a field the feature does not carry", () => { // Left unguarded, the negated operators invert into a match: `!=` becomes // `undefined !== x` and `notIn` becomes "not in the list", both true. One