From d5bad7a1fe8baa57d9dcf8d1cc3d3e133ca310f3 Mon Sep 17 00:00:00 2001 From: Brandon Keepers Date: Thu, 11 Jun 2026 16:38:37 -0400 Subject: [PATCH 1/2] Add PMTiles vector tileset build Adds a packages/tiles workspace that builds a MapLibre-ready PMTiles vector tileset of all stations: - Stations are point features in a single "stations" source-layer. Nested objects (datums, harmonic_constituents, offsets, source, license, epoch) are encoded as JSON-string properties since vector tiles only support scalar values. - Lean properties (id/name/type) at z0-7, full station data at z8-10, merged with tile-join so every tile stays small while no station is ever dropped. - Feature ids are stable 53-bit FNV-1a hashes of station ids for MapLibre setFeatureState. - Built with tippecanoe via the pinned ghcr.io/openwatersio/tippecanoe Docker image; tests decode tiles and round-trip station data against the stations export. - publish.yml attaches the stable-named neaps.pmtiles to releases with the release version embedded in the tileset metadata. --- .github/workflows/publish.yml | 7 +- packages/tiles/README.md | 90 ++++++++++++++++ packages/tiles/build | 44 ++++++++ packages/tiles/build.ts | 50 +++++++++ packages/tiles/docker-compose.yml | 12 +++ packages/tiles/features.ts | 67 ++++++++++++ packages/tiles/package.json | 16 +++ packages/tiles/test/pmtiles.ts | 56 ++++++++++ packages/tiles/test/tiles.test.ts | 174 ++++++++++++++++++++++++++++++ 9 files changed, 515 insertions(+), 1 deletion(-) create mode 100644 packages/tiles/README.md create mode 100755 packages/tiles/build create mode 100644 packages/tiles/build.ts create mode 100644 packages/tiles/docker-compose.yml create mode 100644 packages/tiles/features.ts create mode 100644 packages/tiles/package.json create mode 100644 packages/tiles/test/pmtiles.ts create mode 100644 packages/tiles/test/tiles.test.ts diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ba3f6ad22..eddde83cf 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -38,9 +38,14 @@ jobs: mv packages/tcd/dist/harmonics-metric.tcd "packages/tcd/dist/neaps-${RELEASE_VERSION##*.}-metric.tcd" mv packages/tcd/dist/harmonics-imperial.tcd "packages/tcd/dist/neaps-${RELEASE_VERSION##*.}-imperial.tcd" + - name: Build PMTiles + run: npm run build -w tiles + env: + VERSION: ${{ env.RELEASE_VERSION }} + - name: Create GitHub Release uses: ncipollo/release-action@v1 with: tag: v${{ env.RELEASE_VERSION }} generateReleaseNotes: true - artifacts: "packages/tcd/dist/*.tcd" + artifacts: "packages/tcd/dist/*.tcd,packages/tiles/dist/*.pmtiles" diff --git a/packages/tiles/README.md b/packages/tiles/README.md new file mode 100644 index 000000000..adf1e27f2 --- /dev/null +++ b/packages/tiles/README.md @@ -0,0 +1,90 @@ +# Neaps Tide Database - Vector Tiles (PMTiles) + +This package generates a [PMTiles](https://docs.protomaps.com/pmtiles/) vector tileset of the Neaps Tide Database for use with [MapLibre GL](https://maplibre.org) and other renderers that read Mapbox Vector Tiles. + +> [!WARNING] +> This data is **NOT FOR NAVIGATION**. See the per-station `disclaimers` and `license` properties. + +## Usage + +Download the latest `neaps.pmtiles` from [releases](https://github.com/openwatersio/tide-database/releases) (the stable URL `https://github.com/openwatersio/tide-database/releases/latest/download/neaps.pmtiles` always points at the newest build) and host it on any static file server or CDN that supports HTTP range requests. + +```js +import maplibregl from "maplibre-gl"; +import { Protocol } from "pmtiles"; + +maplibregl.addProtocol("pmtiles", new Protocol().tile); + +map.addSource("tides", { + type: "vector", + url: "pmtiles://https://example.com/neaps.pmtiles", +}); + +map.addLayer({ + id: "tide-stations", + type: "circle", + source: "tides", + "source-layer": "stations", + paint: { + "circle-color": [ + "match", + ["get", "type"], + "reference", + "#1d4ed8", + "#60a5fa", + ], + }, +}); + +map.on("click", "tide-stations", ({ features }) => { + const station = features[0].properties; + // Nested objects are encoded as JSON strings (see "Encoding" below) + const constituents = JSON.parse(station.harmonic_constituents); + const datums = JSON.parse(station.datums); + // ... predict tides with @neaps/tide-predictor +}); +``` + +## Tileset structure + +A single source-layer named `stations` contains every station as a point feature from zoom 0 through 10 (renderers overzoom beyond that). No stations are ever dropped, but properties vary by zoom to keep tiles small: + +- **z0–7 (lean)**: `id`, `name`, `type` — enough to draw and label dots. +- **z8–10 (full)**: everything below. To read full station data, query a tile at z8+. + +## Properties + +| Property | Type | Notes | +| ----------------------- | ----------- | ---------------------------------------------- | +| `id` | string | `/`, e.g. `noaa/9414290` | +| `name` | string | | +| `type` | string | `reference` or `subordinate` | +| `country` | string | Full country name | +| `continent` | string | | +| `region` | string | Optional | +| `timezone` | string | IANA timezone | +| `chart_datum` | string | Key into `datums`, e.g. `MLLW`, `LAT` | +| `disclaimers` | string | Optional | +| `datums` | JSON string | `{ "MLLW": 1.01, "MSL": 2.532, ... }` | +| `harmonic_constituents` | JSON string | `[{ "name": "M2", "amplitude", "phase" },...]` | +| `offsets` | JSON string | Subordinate stations only | +| `source` | JSON string | `{ name, id, published_harmonics, url }` | +| `license` | JSON string | `{ type, commercial_use, url }` | +| `epoch` | JSON string | Optional; `{ start, end }` dates | + +Vector tile properties only support scalar values, so nested objects are encoded as JSON strings — call `JSON.parse()` on them. Subordinate stations include the `datums` and `harmonic_constituents` of their reference station, so each feature is self-sufficient for prediction. + +### Feature ids + +Each feature's numeric id is a stable 53-bit [FNV-1a](https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function) hash of the string station `id` (see `features.ts`), usable with MapLibre's [`setFeatureState`](https://maplibre.org/maplibre-gl-js/docs/API/classes/Map/#setfeaturestate). Ids are consistent across releases. + +## Licensing + +Station data licensing varies by source — check each feature's `license` property (e.g. public domain for NOAA, CC BY 4.0 for TICON-4). Attribution is embedded in the tileset metadata. + +## Contributing + +Building requires Docker (uses the [`ghcr.io/openwatersio/tippecanoe`](https://github.com/openwatersio/tippecanoe) image): + +- Build with `npm run build` — generates NDJSON via `build.ts`, runs tippecanoe for the lean (z0–7) and full (z8–10) variants, and merges them with `tile-join` into `dist/neaps.pmtiles`. +- Test with `npm test`. diff --git a/packages/tiles/build b/packages/tiles/build new file mode 100755 index 000000000..e53b55bd2 --- /dev/null +++ b/packages/tiles/build @@ -0,0 +1,44 @@ +#!/bin/bash + +set -e +cd "$(dirname "$0")" + +VERSION="${VERSION:-$(node -p "require('../../package.json').version")}" + +# Remove any existing files to ensure a clean build +rm -rf dist +mkdir -p dist/tmp + +# Generate newline-delimited GeoJSON for each variant +node build.ts + +# Lean tiles (z0-7): id/name/type only. Drop flags ensure no station is ever +# dropped from a tile, regardless of density. +docker compose run --rm tippecanoe \ + --output=/dist/tmp/stations-lean.pmtiles \ + --layer=stations \ + --minimum-zoom=0 --maximum-zoom=7 \ + --drop-rate=1 --no-feature-limit --no-tile-size-limit \ + --force \ + /dist/stations-lean.ndjson + +# Full tiles (z8-10): all station data, nested objects as JSON strings +docker compose run --rm tippecanoe \ + --output=/dist/tmp/stations-full.pmtiles \ + --layer=stations \ + --minimum-zoom=8 --maximum-zoom=10 \ + --drop-rate=1 --no-feature-limit --no-tile-size-limit \ + --force \ + /dist/stations-full.ndjson + +# Merge into a single tileset with one "stations" layer spanning z0-10 +docker compose run --rm tile-join \ + --output=/dist/neaps.pmtiles \ + --no-tile-size-limit \ + --force \ + --name="Neaps Tide Stations" \ + --description="Tide stations from @neaps/tide-database v${VERSION}. NOT FOR NAVIGATION." \ + --attribution="NOAA; TICON-4" \ + /dist/tmp/stations-lean.pmtiles /dist/tmp/stations-full.pmtiles + +rm -rf dist/tmp diff --git a/packages/tiles/build.ts b/packages/tiles/build.ts new file mode 100644 index 000000000..b506afdbc --- /dev/null +++ b/packages/tiles/build.ts @@ -0,0 +1,50 @@ +#!/usr/bin/env node +/** + * Generates newline-delimited GeoJSON for the vector tile build. Two variants + * are produced: a lean one (id/name/type) for low-zoom tiles and a full one + * (all station data) for high-zoom tiles. The `build` script feeds these to + * tippecanoe and merges the results into a single PMTiles file. + */ + +import { writeFile, mkdir } from "fs/promises"; +import { dirname, join } from "path"; +import { fileURLToPath } from "url"; +import { stations } from "@neaps/tide-database"; +import { featureId, toFeature, type Variant } from "./features.ts"; + +const outDir = join(dirname(fileURLToPath(import.meta.url)), "dist"); + +// Web Mercator latitude limit — tippecanoe silently clips beyond it +const MAX_LATITUDE = 85.0511; + +const sorted = [...stations].sort((a, b) => a.id.localeCompare(b.id)); + +const ids = new Map(); +for (const station of sorted) { + if ( + Math.abs(station.latitude) > MAX_LATITUDE || + Math.abs(station.longitude) > 180 + ) { + throw new Error(`Station ${station.id} is outside Web Mercator bounds`); + } + + const id = featureId(station.id); + const existing = ids.get(id); + if (existing) { + throw new Error( + `Feature id collision between ${existing} and ${station.id}`, + ); + } + ids.set(id, station.id); +} + +await mkdir(outDir, { recursive: true }); + +for (const variant of ["lean", "full"] as Variant[]) { + const ndjson = sorted + .map((station) => JSON.stringify(toFeature(station, variant))) + .join("\n"); + await writeFile(join(outDir, `stations-${variant}.ndjson`), ndjson + "\n"); +} + +console.log(`Wrote ${sorted.length} stations`); diff --git a/packages/tiles/docker-compose.yml b/packages/tiles/docker-compose.yml new file mode 100644 index 000000000..3638ac98e --- /dev/null +++ b/packages/tiles/docker-compose.yml @@ -0,0 +1,12 @@ +services: + tippecanoe: + image: ghcr.io/openwatersio/tippecanoe:2.79.0 + volumes: + - ./dist:/dist + entrypoint: ["tippecanoe"] + + tile-join: + image: ghcr.io/openwatersio/tippecanoe:2.79.0 + volumes: + - ./dist:/dist + entrypoint: ["tile-join"] diff --git a/packages/tiles/features.ts b/packages/tiles/features.ts new file mode 100644 index 000000000..7fea9adad --- /dev/null +++ b/packages/tiles/features.ts @@ -0,0 +1,67 @@ +import type { Station } from "@neaps/tide-database"; + +export type Variant = "lean" | "full"; + +/** + * 64-bit FNV-1a hash of the station id, truncated to 53 bits so it fits in a + * JavaScript safe integer. MVT feature ids must be unsigned integers, and a + * stable hash keeps ids consistent across builds as stations come and go. + */ +export function featureId(id: string): number { + let hash = 0xcbf29ce484222325n; + for (let i = 0; i < id.length; i++) { + hash ^= BigInt(id.charCodeAt(i)); + hash = (hash * 0x100000001b3n) & 0xffffffffffffffffn; + } + return Number(hash & 0x1fffffffffffffn); +} + +type Properties = Record; + +/** + * Vector tile properties only support scalar values, so nested objects + * (datums, harmonic_constituents, offsets, source, license, epoch) are + * encoded as JSON strings for clients to JSON.parse. + */ +function properties(station: Station, variant: Variant): Properties { + const lean: Properties = { + id: station.id, + name: station.name, + type: station.type, + }; + + if (variant === "lean") return lean; + + return { + ...lean, + country: station.country, + continent: station.continent, + timezone: station.timezone, + chart_datum: station.chart_datum, + source: JSON.stringify(station.source), + license: JSON.stringify(station.license), + ...(station.region && { region: station.region }), + ...(station.disclaimers && { disclaimers: station.disclaimers }), + ...(station.datums && + Object.keys(station.datums).length > 0 && { + datums: JSON.stringify(station.datums), + }), + ...(station.harmonic_constituents?.length > 0 && { + harmonic_constituents: JSON.stringify(station.harmonic_constituents), + }), + ...(station.offsets && { offsets: JSON.stringify(station.offsets) }), + ...(station.epoch && { epoch: JSON.stringify(station.epoch) }), + }; +} + +export function toFeature(station: Station, variant: Variant) { + return { + type: "Feature", + id: featureId(station.id), + geometry: { + type: "Point", + coordinates: [station.longitude, station.latitude], + }, + properties: properties(station, variant), + }; +} diff --git a/packages/tiles/package.json b/packages/tiles/package.json new file mode 100644 index 000000000..bdc2ecea7 --- /dev/null +++ b/packages/tiles/package.json @@ -0,0 +1,16 @@ +{ + "name": "tiles", + "private": true, + "type": "module", + "scripts": { + "build": "./build", + "pretest": "npm run build", + "test": "vitest" + }, + "devDependencies": { + "@mapbox/vector-tile": "^2.0.3", + "@neaps/tide-database": "file:../..", + "pbf": "^4.0.1", + "pmtiles": "^4.3.0" + } +} diff --git a/packages/tiles/test/pmtiles.ts b/packages/tiles/test/pmtiles.ts new file mode 100644 index 000000000..5fcba1dfc --- /dev/null +++ b/packages/tiles/test/pmtiles.ts @@ -0,0 +1,56 @@ +import { openSync, readSync } from "fs"; +import { PMTiles, type Source, type RangeResponse } from "pmtiles"; +import { VectorTile } from "@mapbox/vector-tile"; +import Protobuf from "pbf"; + +/** Byte-range source for reading a local .pmtiles file. */ +class FileSource implements Source { + private fd: number; + + constructor(private path: string) { + this.fd = openSync(path, "r"); + } + + getKey(): string { + return this.path; + } + + async getBytes(offset: number, length: number): Promise { + const buffer = Buffer.alloc(length); + readSync(this.fd, buffer, 0, length, offset); + return { + data: buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + length, + ) as ArrayBuffer, + }; + } +} + +export function openPMTiles(path: string): PMTiles { + return new PMTiles(new FileSource(path)); +} + +/** Web Mercator tile coordinates containing the given location. */ +export function lonLatToTile(lon: number, lat: number, zoom: number) { + const n = 2 ** zoom; + const latRad = (lat * Math.PI) / 180; + return { + x: Math.floor(((lon + 180) / 360) * n), + y: Math.floor( + ((1 - Math.log(Math.tan(latRad) + 1 / Math.cos(latRad)) / Math.PI) / 2) * + n, + ), + }; +} + +export function decodeTile(data: ArrayBuffer): VectorTile { + return new VectorTile(new Protobuf(new Uint8Array(data))); +} + +/** All features in a tile's layer, decoded to GeoJSON-ish objects. */ +export function getFeatures(tile: VectorTile, layerName = "stations") { + const layer = tile.layers[layerName]; + if (!layer) return []; + return Array.from({ length: layer.length }, (_, i) => layer.feature(i)); +} diff --git a/packages/tiles/test/tiles.test.ts b/packages/tiles/test/tiles.test.ts new file mode 100644 index 000000000..eba39b51c --- /dev/null +++ b/packages/tiles/test/tiles.test.ts @@ -0,0 +1,174 @@ +import { describe, it, expect, beforeAll } from "vitest"; +import { gzipSync } from "zlib"; +import { join, dirname } from "path"; +import { fileURLToPath } from "url"; +import { stations } from "@neaps/tide-database"; +import type { PMTiles } from "pmtiles"; +import { featureId } from "../features.ts"; +import { + openPMTiles, + lonLatToTile, + decodeTile, + getFeatures, +} from "./pmtiles.ts"; + +const path = join( + dirname(fileURLToPath(import.meta.url)), + "../dist/neaps.pmtiles", +); + +// Named keys (instead of the index signature on decoded MVT properties) so +// the strict noPropertyAccessFromIndexSignature config allows dot access. +type StationProperties = { + id: string; + name: string; + type: string; + country?: string; + continent?: string; + region?: string; + timezone?: string; + chart_datum?: string; + disclaimers?: string; + datums?: string; + harmonic_constituents?: string; + offsets?: string; + source?: string; + license?: string; + epoch?: string; +}; + +let pmtiles: PMTiles; + +beforeAll(() => { + pmtiles = openPMTiles(path); +}); + +// Pick test stations from the export at runtime — quality filtering means +// specific station ids can come and go between data updates. +const reference = stations.find( + (s) => s.type === "reference" && s.harmonic_constituents.length > 0, +)!; +const subordinate = stations.find((s) => s.type === "subordinate")!; + +async function featuresAt(station: typeof reference, zoom: number) { + const { x, y } = lonLatToTile(station.longitude, station.latitude, zoom); + const tile = await pmtiles.getZxy(zoom, x, y); + return getFeatures(decodeTile(tile!.data)); +} + +async function featureAt(station: typeof reference, zoom: number) { + const features = await featuresAt(station, zoom); + return features.find( + (f) => (f.properties as StationProperties).id === station.id, + ); +} + +describe("metadata", () => { + it("has the expected header", async () => { + const header = await pmtiles.getHeader(); + expect(header.minZoom).toBe(0); + expect(header.maxZoom).toBe(10); + expect(header.tileType).toBe(1); // MVT + }); + + it("has name, description, and attribution", async () => { + const metadata = (await pmtiles.getMetadata()) as { + name?: string; + description?: string; + attribution?: string; + }; + expect(metadata.name).toBe("Neaps Tide Stations"); + expect(metadata.description).toMatch(/@neaps\/tide-database v\d/); + expect(metadata.attribution).toContain("NOAA"); + }); + + it("has a single stations layer spanning all zooms", async () => { + const metadata = (await pmtiles.getMetadata()) as { + vector_layers: { id: string; minzoom: number; maxzoom: number }[]; + }; + expect(metadata.vector_layers).toHaveLength(1); + expect(metadata.vector_layers[0]).toMatchObject({ + id: "stations", + minzoom: 0, + maxzoom: 10, + }); + }); +}); + +describe("lean tiles (z0-7)", () => { + it("includes every station in the z0 tile", async () => { + const tile = await pmtiles.getZxy(0, 0, 0); + const features = getFeatures(decodeTile(tile!.data)); + expect(features).toHaveLength(stations.length); + }); + + it("keeps the z0 tile under 500KB compressed", async () => { + const tile = await pmtiles.getZxy(0, 0, 0); + const compressed = gzipSync(Buffer.from(tile!.data)); + expect(compressed.length).toBeLessThan(500_000); + }); + + it("has only lean properties and a hashed feature id", async () => { + const feature = await featureAt(reference, 0); + expect(feature).toBeDefined(); + expect(Object.keys(feature!.properties).sort()).toEqual([ + "id", + "name", + "type", + ]); + expect((feature!.properties as StationProperties).name).toBe( + reference.name, + ); + expect(feature!.id).toBe(featureId(reference.id)); + }); + + it("does not include full data at z7", async () => { + const feature = await featureAt(reference, 7); + expect(feature).toBeDefined(); + expect(feature!.properties).not.toHaveProperty("harmonic_constituents"); + expect(feature!.properties).not.toHaveProperty("datums"); + }); +}); + +describe("full tiles (z8-10)", () => { + it("round-trips a reference station", async () => { + const feature = await featureAt(reference, 10); + expect(feature).toBeDefined(); + + const properties = feature!.properties as StationProperties; + expect(properties.name).toBe(reference.name); + expect(properties.type).toBe("reference"); + expect(properties.country).toBe(reference.country); + expect(properties.timezone).toBe(reference.timezone); + expect(properties.chart_datum).toBe(reference.chart_datum); + expect(feature!.id).toBe(featureId(reference.id)); + + expect(JSON.parse(properties.harmonic_constituents as string)).toEqual( + reference.harmonic_constituents, + ); + expect(JSON.parse(properties.datums as string)).toEqual(reference.datums); + expect(JSON.parse(properties.source as string)).toEqual(reference.source); + expect(JSON.parse(properties.license as string)).toEqual(reference.license); + }); + + it("round-trips a subordinate station", async () => { + const feature = await featureAt(subordinate, 10); + expect(feature).toBeDefined(); + + const properties = feature!.properties as StationProperties; + expect(properties.type).toBe("subordinate"); + expect(JSON.parse(properties.offsets as string)).toEqual( + subordinate.offsets, + ); + expect(JSON.parse(properties.harmonic_constituents as string)).toEqual( + subordinate.harmonic_constituents, + ); + expect(JSON.parse(properties.datums as string)).toEqual(subordinate.datums); + }); + + it("includes full data starting at z8", async () => { + const feature = await featureAt(reference, 8); + expect(feature).toBeDefined(); + expect(feature!.properties).toHaveProperty("harmonic_constituents"); + }); +}); From 24ba42e4b71b4217530e2467367430173b314f43 Mon Sep 17 00:00:00 2001 From: Brandon Keepers Date: Sat, 13 Jun 2026 08:20:54 -0400 Subject: [PATCH 2/2] Add mbtiles build --- .github/workflows/publish.yml | 2 +- packages/tiles/build | 21 ++++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index eddde83cf..d585fb69a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -48,4 +48,4 @@ jobs: with: tag: v${{ env.RELEASE_VERSION }} generateReleaseNotes: true - artifacts: "packages/tcd/dist/*.tcd,packages/tiles/dist/*.pmtiles" + artifacts: "packages/tcd/dist/*.tcd,packages/tiles/dist/*.pmtiles,packages/tiles/dist/*.mbtiles" diff --git a/packages/tiles/build b/packages/tiles/build index e53b55bd2..df8fee1c4 100755 --- a/packages/tiles/build +++ b/packages/tiles/build @@ -31,14 +31,17 @@ docker compose run --rm tippecanoe \ --force \ /dist/stations-full.ndjson -# Merge into a single tileset with one "stations" layer spanning z0-10 -docker compose run --rm tile-join \ - --output=/dist/neaps.pmtiles \ - --no-tile-size-limit \ - --force \ - --name="Neaps Tide Stations" \ - --description="Tide stations from @neaps/tide-database v${VERSION}. NOT FOR NAVIGATION." \ - --attribution="NOAA; TICON-4" \ - /dist/tmp/stations-lean.pmtiles /dist/tmp/stations-full.pmtiles +# Merge into a single tileset with one "stations" layer spanning z0-10. +# tile-join picks the format from the output extension. +for ext in pmtiles mbtiles; do + docker compose run --rm tile-join \ + --output=/dist/neaps.${ext} \ + --no-tile-size-limit \ + --force \ + --name="Neaps Tide Stations" \ + --description="Tide stations from @neaps/tide-database v${VERSION}. NOT FOR NAVIGATION." \ + --attribution="NOAA; TICON-4" \ + /dist/tmp/stations-lean.pmtiles /dist/tmp/stations-full.pmtiles +done rm -rf dist/tmp