diff --git a/docs/source/maps/source_tab.rst b/docs/source/maps/source_tab.rst index 937e1ddb..3dbcb3cb 100644 --- a/docs/source/maps/source_tab.rst +++ b/docs/source/maps/source_tab.rst @@ -305,6 +305,39 @@ The Zarr source renders a raster layer from a public `Zarr `_ ------------------------------------------------------------------------------------------------------------------------ +++++++++++ +GeoPackage +++++++++++ + +The GeoPackage source renders **vector** layers from a public `GeoPackage `_ (``.gpkg``) file. A GeoPackage is an OGC SQLite database that may hold several feature tables, so you supply a file URL **and** the table (layer) name — one map layer per table. TethysDash reads the file entirely in the browser and reprojects it to the map projection, so no server-side conversion is involved. Layers are styled and queried like a GeoJSON layer. + +**Layer Properties:** + - **url:** (required) Public URL of the GeoPackage file (an ``https`` URL, or an ``s3://`` URL which is translated to its public ``https`` form). The host must allow cross-origin (CORS) reads. + - **layer:** (required) Name of the feature table within the GeoPackage to render. + +.. note:: + The file is downloaded and parsed in the browser, so this suits moderate feature counts; very large tables may be slow to render. Feature tables in projected CRSs (e.g. UTM) are reprojected automatically; other custom CRSs may not be supported. + +**Example JSON Configuration:** + +:: + + { + "type": "VectorLayer", + "props": { + "name": "Buildings", + "source": { + "type": "GeoPackage", + "props": { + "url": "https://example.com/flood_event.gpkg", + "layer": "buildings" + } + } + } + } + +------------------------------------------------------------------------------------------------------------------------ + +++++++++++++ Custom Layers +++++++++++++ diff --git a/package-lock.json b/package-lock.json index e07795ef..18992cb3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,11 +40,12 @@ "html-react-parser": "^5.1.18", "json5": "^2.2.3", "ol": "10.4.0", + "ol-load-geopackage": "^2.3.1", "ol-mapbox-style": "12.4.0", "ol-pmtiles": "^2.0.2", "plotly.js-strict-dist-min": "^2.35.2", "prismjs": "^1.28.0", - "proj4": "2.21.0", + "proj4": "^2.21.0", "rc-slider": "^11.1.8", "react": "^18.3.1", "react-bootstrap": "^2.10.2", @@ -67,6 +68,7 @@ "sass-loader": "^12.3.0", "shapefile": "0.6.6", "simple-xml-to-json": "^1.2.3", + "sql.js": "1.13.0", "style-loader": "^3.3.1", "styled-components": "^6.3.9", "swiper": "^11.2.1", @@ -17487,6 +17489,16 @@ "url": "https://opencollective.com/openlayers" } }, + "node_modules/ol-load-geopackage": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/ol-load-geopackage/-/ol-load-geopackage-2.3.1.tgz", + "integrity": "sha512-ijyxZ/09j7a4fTTZOSg7gzm2u0p1oBIOCZY7PB3G8WDb49WkRTc9Sk1ewGiBNs/68Qavb1jFEjrm9ZloHmddAg==", + "license": "ISC", + "dependencies": { + "ol": ">=6.7.0", + "sql.js": "1.13.0" + } + }, "node_modules/ol-mapbox-style": { "version": "12.4.0", "resolved": "https://registry.npmjs.org/ol-mapbox-style/-/ol-mapbox-style-12.4.0.tgz", @@ -20744,6 +20756,12 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/sql.js": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/sql.js/-/sql.js-1.13.0.tgz", + "integrity": "sha512-RJbVP1HRDlUUXahJ7VMTcu9Rm1Nzw+EBpoPr94vnbD4LwR715F3CcxE2G2k45PewcaZ57pjetYa+LoSJLAASgA==", + "license": "MIT" + }, "node_modules/stack-trace": { "version": "0.0.9", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz", diff --git a/package.json b/package.json index 22d877c9..239cc880 100644 --- a/package.json +++ b/package.json @@ -50,11 +50,12 @@ "html-react-parser": "^5.1.18", "json5": "^2.2.3", "ol": "10.4.0", + "ol-load-geopackage": "^2.3.1", "ol-mapbox-style": "12.4.0", "ol-pmtiles": "^2.0.2", "plotly.js-strict-dist-min": "^2.35.2", "prismjs": "^1.28.0", - "proj4": "2.21.0", + "proj4": "^2.21.0", "rc-slider": "^11.1.8", "react": "^18.3.1", "react-bootstrap": "^2.10.2", @@ -77,6 +78,7 @@ "sass-loader": "^12.3.0", "shapefile": "0.6.6", "simple-xml-to-json": "^1.2.3", + "sql.js": "1.13.0", "style-loader": "^3.3.1", "styled-components": "^6.3.9", "swiper": "^11.2.1", @@ -224,7 +226,7 @@ "/node_modules" ], "moduleNameMapper": { - "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "/reactapp/__tests__/utilities/fileMock.js", + "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga|wasm)$": "/reactapp/__tests__/utilities/fileMock.js", "^.+\\.module\\.(css|sass|scss|less)$": "identity-obj-proxy", "\\.(css|less|scss|sss|styl)$": "/node_modules/jest-css-modules" }, diff --git a/reactapp/__tests__/components/map/ModuleLoader.test.js b/reactapp/__tests__/components/map/ModuleLoader.test.js index cb61c805..9e7428fe 100644 --- a/reactapp/__tests__/components/map/ModuleLoader.test.js +++ b/reactapp/__tests__/components/map/ModuleLoader.test.js @@ -15,6 +15,10 @@ import moduleLoader, { withAutoCrossOrigin, zarrSourceToGeoTIFF, applyAutoRamp, + loadGeoPackage, + s3UrlToHttps, + registerGeoPackageProjections, + GeoPackageError, } from "components/map/ModuleLoader"; import { fromUrl } from "geotiff"; import WebGLTile from "ol/layer/WebGLTile.js"; @@ -52,9 +56,18 @@ import { defaultStroke, defaultStrokeWidth, } from "components/inputs/RuleEditor.js"; +import { get as getProjection } from "ol/proj"; +import proj4 from "proj4"; +import { loadGpkg } from "ol-load-geopackage"; jest.mock("geotiff", () => ({ fromUrl: jest.fn() })); +jest.mock("ol-load-geopackage", () => ({ + __esModule: true, + initSqlJsWasm: jest.fn(), + loadGpkg: jest.fn(), +})); + jest.mock("ol/source/GeoTIFF.js", () => { const ActualSource = jest.requireActual("ol/source/Source.js").default; const spy = jest.fn(); @@ -2721,6 +2734,149 @@ describe("applyAutoRamp", () => { }); }); +describe("s3UrlToHttps", () => { + test("passes an https url through unchanged", () => { + expect(s3UrlToHttps("https://x/y.gpkg")).toBe("https://x/y.gpkg"); + }); + + test("returns non-string input unchanged", () => { + expect(s3UrlToHttps(undefined)).toBeUndefined(); + }); + + test("converts s3:// to a virtual-hosted https url, sniffing the region", () => { + expect( + s3UrlToHttps("s3://cog-s3-test-1234-us-east-1-an/Guatemala_IBF/f.gpkg"), + ).toBe( + "https://cog-s3-test-1234-us-east-1-an.s3.us-east-1.amazonaws.com/Guatemala_IBF/f.gpkg", + ); + }); + + test("honors a region embedded in the bucket name", () => { + expect(s3UrlToHttps("s3://data-eu-west-2-x/k.gpkg")).toBe( + "https://data-eu-west-2-x.s3.eu-west-2.amazonaws.com/k.gpkg", + ); + }); + + test("defaults the region when none is in the bucket name", () => { + expect(s3UrlToHttps("s3://my-bucket/a/b.gpkg")).toBe( + "https://my-bucket.s3.us-east-1.amazonaws.com/a/b.gpkg", + ); + }); + + test("handles an s3 url with no key", () => { + expect(s3UrlToHttps("s3://plain-bucket")).toBe( + "https://plain-bucket.s3.us-east-1.amazonaws.com/", + ); + }); +}); + +describe("registerGeoPackageProjections", () => { + test("registers WGS84 UTM zones with proj4 and OpenLayers", () => { + registerGeoPackageProjections(); + expect(proj4.defs("EPSG:32615")).toBeTruthy(); + expect(proj4.defs("EPSG:32715")).toBeTruthy(); + expect(getProjection("EPSG:32615")).not.toBeNull(); + }); + + test("is idempotent", () => { + expect(() => { + registerGeoPackageProjections(); + registerGeoPackageProjections(); + }).not.toThrow(); + }); +}); + +describe("loadGeoPackage", () => { + test("returns the requested table's vector source", async () => { + const src = new VectorSource(); + loadGpkg.mockResolvedValue([{ roads: src }, {}]); + const out = await loadGeoPackage( + { props: { url: "https://h/t1.gpkg", layer: "roads" } }, + "EPSG:3857", + ); + expect(out).toBe(src); + expect(loadGpkg).toHaveBeenCalledWith("https://h/t1.gpkg", "EPSG:3857"); + }); + + test("translates an s3:// url before loading", async () => { + loadGpkg.mockResolvedValue([{ t: new VectorSource() }, {}]); + await loadGeoPackage( + { props: { url: "s3://b-us-east-1-x/t2.gpkg", layer: "t" } }, + "EPSG:3857", + ); + expect(loadGpkg).toHaveBeenCalledWith( + "https://b-us-east-1-x.s3.us-east-1.amazonaws.com/t2.gpkg", + "EPSG:3857", + ); + }); + + test("rejects when the url is missing", async () => { + await expect( + loadGeoPackage({ props: { layer: "t" } }, "EPSG:3857"), + ).rejects.toThrow(GeoPackageError); + expect(loadGpkg).not.toHaveBeenCalled(); + }); + + test("rejects when the table name is missing", async () => { + await expect( + loadGeoPackage({ props: { url: "https://h/t3.gpkg" } }, "EPSG:3857"), + ).rejects.toThrow(GeoPackageError); + }); + + test("rejects an unknown table, listing the available ones", async () => { + loadGpkg.mockResolvedValue([ + { roads: new VectorSource(), bldgs: new VectorSource() }, + {}, + ]); + await expect( + loadGeoPackage( + { props: { url: "https://h/t4.gpkg", layer: "nope" } }, + "EPSG:3857", + ), + ).rejects.toThrow(/roads|bldgs/); + }); + + test("parses a file only once across layers (cache by url+projection)", async () => { + loadGpkg.mockResolvedValue([{ t: new VectorSource() }, {}]); + const cfg = { props: { url: "https://h/t5.gpkg", layer: "t" } }; + await loadGeoPackage(cfg, "EPSG:3857"); + await loadGeoPackage(cfg, "EPSG:3857"); + expect(loadGpkg).toHaveBeenCalledTimes(1); + }); + + test("does not cache a failed load", async () => { + loadGpkg + .mockRejectedValueOnce(new Error("boom")) + .mockResolvedValue([{ t: new VectorSource() }, {}]); + const cfg = { props: { url: "https://h/t6.gpkg", layer: "t" } }; + await expect(loadGeoPackage(cfg, "EPSG:3857")).rejects.toThrow("boom"); + const out = await loadGeoPackage(cfg, "EPSG:3857"); + expect(out).toBeInstanceOf(VectorSource); + expect(loadGpkg).toHaveBeenCalledTimes(2); + }); + + test("caches separately per display projection", async () => { + loadGpkg.mockResolvedValue([{ t: new VectorSource() }, {}]); + const cfg = { props: { url: "https://h/t7.gpkg", layer: "t" } }; + await loadGeoPackage(cfg, "EPSG:3857"); + await loadGeoPackage(cfg, "EPSG:4326"); + expect(loadGpkg).toHaveBeenCalledTimes(2); + }); + + test("moduleLoader routes a GeoPackage config to loadGeoPackage", async () => { + const src = new VectorSource(); + loadGpkg.mockResolvedValue([{ roads: src }, {}]); + const out = await moduleLoader( + { + type: "GeoPackage", + props: { url: "https://h/t8.gpkg", layer: "roads" }, + }, + "EPSG:3857", + ); + expect(out).toBe(src); + }); +}); + describe("matchesCondition — a field the feature does not carry", () => { // Left unguarded, the negated operators invert into a match: `!=` becomes // `undefined !== x` and `notIn` becomes "not in the list", both true. One diff --git a/reactapp/components/map/ModuleLoader.js b/reactapp/components/map/ModuleLoader.js index 2aeb2197..22b0c86f 100644 --- a/reactapp/components/map/ModuleLoader.js +++ b/reactapp/components/map/ModuleLoader.js @@ -38,6 +38,9 @@ import { buildCategoricalStyleColor, isUsableClass, } from "components/map/geoTIFFStyle"; +import proj4 from "proj4"; +import { register as registerProj4 } from "ol/proj/proj4.js"; +import sqlWasmUrl from "sql.js/dist/sql-wasm.wasm"; const moduleCache = {}; const styleCache = new Map(); @@ -388,7 +391,93 @@ export async function applyAutoRamp(layerConfig) { return layerConfig; } +export class GeoPackageError extends Error {} + +// s3://bucket/key -> virtual-hosted https so the browser can fetch it directly. +export function s3UrlToHttps(url, defaultRegion = "us-east-1") { + if (typeof url !== "string" || !url.startsWith("s3://")) return url; + const rest = url.slice(5); + const slash = rest.indexOf("/"); + const bucket = slash === -1 ? rest : rest.slice(0, slash); + const key = slash === -1 ? "" : rest.slice(slash + 1); + const region = + bucket.match(/(us|eu|ap|sa|ca|me|af)-[a-z]+-\d+/)?.[0] ?? defaultRegion; + return `https://${bucket}.s3.${region}.amazonaws.com/${key}`; +} + +// Register proj4 + all WGS84 UTM zones so ol can reproject projected GeoPackages. +let projectionsRegistered = false; +export function registerGeoPackageProjections() { + if (projectionsRegistered) return; + for (let zone = 1; zone <= 60; zone++) { + proj4.defs( + `EPSG:${32600 + zone}`, + `+proj=utm +zone=${zone} +datum=WGS84 +units=m +no_defs +type=crs`, + ); + proj4.defs( + `EPSG:${32700 + zone}`, + `+proj=utm +zone=${zone} +south +datum=WGS84 +units=m +no_defs +type=crs`, + ); + } + registerProj4(proj4); + projectionsRegistered = true; +} + +// Lazy-load ol-load-geopackage and init the sql.js wasm loader once. +let geoPackageLib = null; +async function getGeoPackageLib() { + if (!geoPackageLib) { + const lib = await import("ol-load-geopackage"); + lib.initSqlJsWasm(sqlWasmUrl.replace(/\/sql-wasm\.wasm$/, "")); + geoPackageLib = lib; + } + return geoPackageLib; +} + +// Cache parsed gpkg per url+projection; one file often backs several layers. +const geoPackageCache = new Map(); + +// Read one GeoPackage table in-browser as a reprojected OL vector source. +export async function loadGeoPackage(config, mapProjection) { + const rawUrl = config.props?.url; + const table = config.props?.layer; + if (!rawUrl) { + throw new GeoPackageError("GeoPackage source requires a file URL"); + } + if (!table) { + throw new GeoPackageError("GeoPackage source requires a table name"); + } + + const url = s3UrlToHttps(rawUrl); + registerGeoPackageProjections(); + const { loadGpkg } = await getGeoPackageLib(); + + const cacheKey = `${url}::${mapProjection}`; + if (!geoPackageCache.has(cacheKey)) { + geoPackageCache.set(cacheKey, loadGpkg(url, mapProjection)); + } + let dataByTable; + try { + [dataByTable] = await geoPackageCache.get(cacheKey); + } catch (error) { + geoPackageCache.delete(cacheKey); + throw error; + } + + const source = dataByTable[table]; + if (!source) { + throw new GeoPackageError( + `Table "${table}" not found in GeoPackage. Available tables: ` + + Object.keys(dataByTable).join(", "), + ); + } + return source; +} + const moduleLoader = async (config, mapProjection, getMapProjection) => { + if (config.type === "GeoPackage") { + return loadGeoPackage(config, mapProjection); + } if (config.type === "Zarr") { // Already yields OL's `sources` shape, so it skips the GeoTIFF branch below. config = zarrSourceToGeoTIFF(config); diff --git a/reactapp/components/map/utilities.js b/reactapp/components/map/utilities.js index c2b0b8ab..ba742251 100644 --- a/reactapp/components/map/utilities.js +++ b/reactapp/components/map/utilities.js @@ -25,6 +25,7 @@ import Protobuf from "pbf"; export const CLIENT_VECTOR_SOURCE_TYPES = [ "GeoJSON", "ESRI Feature Service", + "GeoPackage", "Shapefile", ]; @@ -175,6 +176,13 @@ export const sourcePropertiesOptions = { mask_below: { placeholder: "Mask values at or below this" }, }, }, + GeoPackage: { + required: { + url: { placeholder: "GeoPackage file URL (https or s3)" }, + layer: { placeholder: "Table (layer) name" }, + }, + optional: {}, + }, "Vector Tile": { required: { urls: { diff --git a/reactapp/config/webpack.config.js b/reactapp/config/webpack.config.js index 86c42539..01ca35f1 100644 --- a/reactapp/config/webpack.config.js +++ b/reactapp/config/webpack.config.js @@ -35,6 +35,8 @@ module.exports = (env, argv) => { path.resolve(__dirname, "../"), path.resolve(__dirname, "../../node_modules"), ], + // sql.js references node core modules in its node path; stub for browser. + fallback: { fs: false, path: false, crypto: false }, }, plugins: [ new Dotenv({ @@ -110,6 +112,12 @@ module.exports = (env, argv) => { }, ], }, + { + // Emit sql-wasm.wasm verbatim so ol-load-geopackage can fetch it. + test: /sql-wasm\.wasm$/, + type: "asset/resource", + generator: { filename: "sql-wasm.wasm" }, + }, ], }, optimization: {