diff --git a/.gitignore b/.gitignore index b146c1754..79e6e57be 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ node_modules dist package-lock.json tmp + +# generated by scripts/generate-pack.mjs +src/generated/ diff --git a/docs/lazy-loading.md b/docs/lazy-loading.md new file mode 100644 index 000000000..98f8d08c3 --- /dev/null +++ b/docs/lazy-loading.md @@ -0,0 +1,224 @@ +# Off-heap prediction-data pack + +Status: **implemented.** Prediction data (harmonic constituents, datums, epoch) +now lives in a `stations.pack` file. The Node build reads only the bytes for the +station being loaded, holding nothing resident; the browser build bundles the +records. The public API is unchanged and still synchronous. The two builds are +selected by `exports` conditions. + +## Problem + +`src/stations.ts` previously used `import.meta.glob(..., { eager: true })` to +inline **every** station's JSON into the bundle, and any import realized all of +it as live JS objects — even a health check or a single-station lookup. + +Measured cost — two symptoms, one cause: + +- **CPU:** importing `@neaps/tide-database` was ~358 ms; a warm prediction is + ~2 ms. On a platform that cold-starts frequently (the tides API burns ~95% of + its CPU budget), that module evaluation _is_ the bill. +- **Memory:** parsing all 8,290 stations (filtered to ~6,177 quality) into live + JS objects cost **118 MB of heap / 663 MB RSS** (the predictor is 4 MB — the + database is all of it). This OOMs memory-constrained devices: signalk-tides + ([#103](https://github.com/openwatersio/signalk-tides/issues/103)) crashes a + Victron Cerbo GX with a V8 "Reached heap limit" error, because the 118 MB + baseline consumes the constrained heap's headroom. The prediction hot loop + itself does **not** leak — measured flat over 600 iterations (~10 h of runtime). + +## Key insight: don't put the data on the heap + +The V8 "Reached heap limit" OOM is governed by `heapUsed` vs +`--max-old-space-size`. The previous build kept all 8,290 records as JavaScript +values on that heap. Anything bundled into JavaScript lands there — measured on +the 20 MB pack: + +| Form | resident cost | +| ----------------------------------------- | --------------------------------------- | +| 8,290 JSON string literals (previous) | ~69 MB heap | +| one base64 string literal, decoded | ~58 MB heap (OOMs under a 48 MB cap) | +| **read by byte range from a file** (this) | **~2 MB external**, nothing on the heap | + +String and base64 literals live in the module's constant pool — on the heap. So +the data ships as a **file**, and the Node build reads only the record for the +station being loaded (`openSync` once, `readSync` its byte range), holding +nothing resident: the OS page-caches the touched pages and can evict them under +pressure. (`readFileSync`-ing the whole pack into a `Buffer` would also stay off +the V8 heap — Buffers are external memory — but it pins ~20 MB resident; reading +per record avoids even that, so `external` stays ~2 MB.) + +## What ships + +- **Metadata** — inlined into the JS as object literals via a build macro + (`createStationMeta`): identity plus `offsets`, `source`, `license`, + `chart_datum`, etc. Everything except the prediction data. +- **`stations.pack`** — the prediction data (`harmonic_constituents`, `datums`, + `epoch`) per station, concatenated as UTF-8 JSON records. Shipped as a package + asset (`dist/node/generated/stations.pack`). +- **A byte-range index** — `id -> [offset, length]` into the pack, generated + alongside the pack and bundled into the JS (`src/generated/pack-index.ts`), so + the reader is dependency-free: slice `pack[offset .. offset+length]` → + `JSON.parse`. Offsets are UTF-8 **bytes** (station names carry multibyte chars). + +## Two builds: Node and browser + +The package is publicly distributed and may be used in a browser, which has no +filesystem. So the prediction-data source is swapped per build behind the +`#station-data` subpath import: + +```jsonc +// package.json +"imports": { + "#station-data": { + "browser": "./src/station-data.browser.ts", // bundled JSON strings + "default": "./src/station-data.ts" // per-record pack reads (Node) + } +}, +"exports": { + ".": { + "browser": "./dist/browser/index.js", + "node": "./dist/node/index.js", + "default": "./dist/browser/index.js" + } +} +``` + +- **Node** (`station-data.ts`): `openSync` the pack once; `getData(id)` `readSync`s + only that record's byte range and parses it. Nothing resident — `external` stays + ~2 MB and the parsed record is transient. +- **Browser** (`station-data.browser.ts`): the records are bundled as JSON strings + (via the `createStationDataById` macro) and parsed one at a time. This costs + more heap (the strings are on the JS heap), but browsers have no equivalent of + Node's tight `--max-old-space-size` limit and no filesystem to read. + +Both are **ESM only** and expose the same synchronous `getData(id)`, so +`stations.ts` and everything downstream is identical across builds. tsdown builds +both entries; the browser build never contains `node:fs`. (No CJS build — see +below.) + +## The reader and subordinate stations + +`stations.ts` builds `allStations` from the metadata, attaching lazy getters for +the prediction fields: + +```ts +function makeStation(m: StationMeta): Station { + // Subordinate stations predict from their reference's data, applying their own + // offsets; resolve to the reference's record. + const dataId = + m.type === "subordinate" && m.offsets ? m.offsets.reference : m.id; + const station = { ...m } as Station; + Object.defineProperties(station, { + harmonic_constituents: { + enumerable: true, + get: () => getData(dataId).harmonic_constituents, + }, + datums: { enumerable: true, get: () => getData(dataId).datums }, + epoch: { enumerable: true, get: () => getData(m.id).epoch }, + }); + return station; +} +``` + +No caching — a persistent cache on these module-level objects would pull the data +back onto the heap on a process that touches every station. Reading a record is a +`readSync` of its byte range + `JSON.parse` (~µs, and the file is OS-page-cached +after warmup). + +## Search indexes (geo + text) + +| Index | Serialized | Built in memory | Used by | Plan | +| ----------------- | ---------- | --------------- | --------------------- | ---------------------------- | +| geo (KDBush) | ~66 KB | ~66 KB | near / nearest / bbox | bundled, eager — negligible | +| text (MiniSearch) | ~1.5 MB | ~15 MB | `search()` only | built lazily on first search | + +`near`/`nearest`/`bbox` resolve coordinates to station ids from the geo index +without touching prediction data. The text index is built on the first `search()` +call, so geo/id-only consumers (like the plugin) never pay its ~15 MB. + +## Build pipeline + +`npm run build`: + +1. `generate` (`scripts/generate-pack.mjs`) — reads `data/**/*.json`, writes + `src/generated/stations.pack` and `src/generated/pack-index.ts` (both + git-ignored). A `pretest` hook runs it too, so the tests and both builds always + have a current pack. +2. `tsdown` — builds `dist/node` and `dist/browser` (both ESM), resolving + `#station-data` per build. +3. `copy-pack` — copies the pack to `dist/node/generated/` so the runtime + `new URL("./generated/stations.pack", import.meta.url)` resolves. +4. `tsc --noEmit` — type-checks src and the tests/examples against the built types. +5. `smoke` (`scripts/smoke.mjs`) — imports the built node and browser ESM entries, + checks a reference and a subordinate station resolve their prediction data, and + asserts the browser bundle has no `node:fs`. Runs on every build so a broken + artifact can't ship. + +## Results + +Node build, import + one `nearest()`, GC'd: + +| | `heapUsed` | `external` | +| ------------------------- | ----------- | ---------- | +| eager (before) | 118 MB | — | +| bundled strings (interim) | 69 MB | — | +| **per-record pack (now)** | **35.8 MB** | 1.9 MB | + +The remaining ~36 MB is the sync API's own cost — 8,290 `Station` objects plus +full metadata and the id maps — not the prediction data, which is never resident +(`external` is ~2 MB, and touching all 8,290 stations keeps it flat). It +reads all 8,290 stations under `--max-old-space-size=40`; the old build needs +~69 MB just to start. All 31,862 tests pass. The browser build works with the +records bundled and contains no `node:fs`. + +Trimming metadata to bare identity (deriving `source.id` from the +`/` id, moving `source`/`license`/`offsets` into the pack) +would cut the ~36 MB further, at the cost of extra reads for detail/subordinate +resolution. Left as a future optimization. + +## Garbage collection + +GC depends on reachability, not on an operation being called "lazy." + +- The metadata objects, `allStations`, `stations`, and `stationsById` are + module-level and live for the process; the pack itself is never held in memory — + only an open file descriptor and transient per-record read buffers. +- A getter parses one record and returns it. Because getters do not cache, the + parsed object is collectible once the caller releases it; walking every station + without retaining results stays flat. +- A bounded LRU cache could be added later if repeated reads ever measure as hot. + An unbounded `Map` would recreate the original problem by pulling data back onto + the heap. +- After collection V8 keeps heap pages reserved, so RSS may not fall even though + the memory is reusable — but the OOM is a heap-limit error, and `heapUsed` is + what dropped. + +## Alternatives considered + +| Approach | Verdict | +| ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Indexed pack file, read per record (`readSync`) | **Chosen.** Sync API, nothing resident (~2 MB external), one small dependency-free reader. | +| Indexed pack file, whole file in a Buffer | Off-heap too, but pins ~20 MB resident; per-record read avoids it. | +| Bundle data as JSON string literals (interim "Phase 0") | Works but ~69 MB heap — the strings sit on the heap. | +| Bundle data as one base64 literal → Buffer | Rejected: the base64 literal is on-heap (~58 MB, OOMs under a 48 MB cap). | +| `import.meta.glob({ eager: false })` (dynamic imports) | Tested: 8 MB baseline, but emits 8,290 chunk files, bundles to ~31 MB on Workers, and the module cache climbs to ~19 MB after 1,000 distinct loads and never releases. | +| SQLite (better-sqlite3 / D1 / sql.js) | Overkill for read-only id→blob; native/WASM weight across runtimes. | +| One large JSON object | Parsing materializes the entire dataset. | + +## Future: async / edge source + +The current design is synchronous and covers Node (off-heap pack file) and the +browser (bundled records). It does **not** need an async API. If a future consumer +wants low memory in an environment with neither a filesystem nor room to bundle +20 MB — e.g. the tides API on Cloudflare Workers pulling records from R2 — the same +pack can be served by HTTP/R2 range reads behind an **async** `getStation(id)` +(a new `@neaps/tide-database/async` entry). That would be additive; it is not +required by the Node or browser builds and is deferred until a consumer needs it. + +## ESM only (no CJS build) + +`kdbush` and `geokdbush` are ESM-only packages (no `require` export), so a CJS +build can't `require()` them without a double-wrapped-default interop bug +(`KDBush.from is not a function`). All first-party consumers use ESM, so the +package ships **ESM only** — the `require` condition is removed from `exports`. +Modern Node still lets `require()` load the ESM entry (require-of-ESM); older +CJS-only tooling would need to `import()` it. diff --git a/package.json b/package.json index ec2c6341d..0a33b8eca 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@neaps/tide-database", - "version": "0.8.0", + "version": "0.9.0", "description": "A public database of tide harmonics", "keywords": [ "tides", @@ -18,13 +18,20 @@ "license": "MIT", "author": "Brandon Keepers ", "type": "module", - "main": "./dist/index.cjs", - "module": "./dist/index.js", - "types": "./dist/index.d.cts", + "main": "./dist/node/index.js", + "module": "./dist/node/index.js", + "types": "./dist/node/index.d.ts", + "imports": { + "#station-data": { + "browser": "./src/station-data.browser.ts", + "default": "./src/station-data.ts" + } + }, "exports": { ".": { - "import": "./dist/index.js", - "require": "./dist/index.cjs" + "browser": "./dist/browser/index.js", + "node": "./dist/node/index.js", + "default": "./dist/browser/index.js" }, "./package.json": "./package.json" }, @@ -32,8 +39,11 @@ "doc": "docs" }, "scripts": { - "build": "tsc -b && tsc -p tsconfig.node.json && tsdown", + "generate": "node scripts/generate-pack.mjs", + "build": "npm run generate && tsdown && node scripts/copy-pack.mjs && tsc -p tsconfig.json --noEmit && tsc -p tsconfig.node.json && npm run smoke", + "smoke": "node scripts/smoke.mjs", "prepare": "npm run build", + "pretest": "npm run generate", "test": "vitest -r test", "lint": "prettier --check .", "format": "prettier --write ." diff --git a/scripts/copy-pack.mjs b/scripts/copy-pack.mjs new file mode 100644 index 000000000..45dc729e7 --- /dev/null +++ b/scripts/copy-pack.mjs @@ -0,0 +1,14 @@ +// Copies the generated pack next to the bundled Node entry so the runtime +// `new URL("./generated/stations.pack", import.meta.url)` resolves in dist. +// Only the Node build reads the pack; the browser build bundles the data. +import { mkdirSync, copyFileSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +mkdirSync(join(root, "dist", "node", "generated"), { recursive: true }); +copyFileSync( + join(root, "src", "generated", "stations.pack"), + join(root, "dist", "node", "generated", "stations.pack"), +); +console.log("copied stations.pack -> dist/node/generated/"); diff --git a/scripts/generate-pack.mjs b/scripts/generate-pack.mjs new file mode 100644 index 000000000..c152d0129 --- /dev/null +++ b/scripts/generate-pack.mjs @@ -0,0 +1,76 @@ +// Generates the prediction-data pack the Node build reads at runtime. +// +// Emits two files under src/generated/ (git-ignored, regenerated on build/test): +// stations.pack concatenated UTF-8 JSON records {harmonic_constituents, datums, epoch} +// pack-index.ts id -> [byteOffset, byteLength] into the pack +// +// The pack ships as a package asset and is loaded into an off-heap Buffer, so +// importing tide-database no longer holds every station's prediction data on the +// V8 heap. Offsets are UTF-8 byte offsets (station names carry multibyte chars). + +import { readFileSync, writeFileSync, mkdirSync, readdirSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +const dataDir = join(root, "data"); +const outDir = join(root, "src", "generated"); + +function walk(dir) { + return readdirSync(dir, { withFileTypes: true }).flatMap((e) => { + const p = join(dir, e.name); + return e.isDirectory() ? walk(p) : p.endsWith(".json") ? [p] : []; + }); +} + +// Sorted for a deterministic pack layout. +const files = walk(dataDir).sort(); + +const chunks = []; +const index = {}; +const references = []; +let offset = 0; + +for (const file of files) { + const id = file.slice(dataDir.length + 1).replace(/\.json$/, ""); + const s = JSON.parse(readFileSync(file, "utf8")); + if (s.type === "subordinate" && s.offsets?.reference) { + references.push([id, s.offsets.reference]); + } + const record = JSON.stringify({ + harmonic_constituents: s.harmonic_constituents ?? [], + datums: s.datums ?? {}, + epoch: s.epoch, + }); + const buf = Buffer.from(record, "utf8"); + index[id] = [offset, buf.length]; + chunks.push(buf); + offset += buf.length; +} + +// Fail the build if any subordinate points at a missing reference — otherwise it +// would only surface at prediction time as a runtime error. +for (const [id, reference] of references) { + if (!(reference in index)) { + throw new Error( + `Station ${id} references missing reference station ${reference}`, + ); + } +} + +const pack = Buffer.concat(chunks); + +mkdirSync(outDir, { recursive: true }); +writeFileSync(join(outDir, "stations.pack"), pack); +writeFileSync( + join(outDir, "pack-index.ts"), + "// Generated by scripts/generate-pack.mjs — do not edit.\n" + + "export const packIndex: Record = " + + JSON.stringify(index) + + ";\n", +); + +console.log( + `generated pack: ${(pack.length / 1048576).toFixed(1)} MB, ${files.length} stations, ` + + `index ${(Buffer.byteLength(JSON.stringify(index)) / 1048576).toFixed(2)} MB`, +); diff --git a/scripts/smoke.mjs b/scripts/smoke.mjs new file mode 100644 index 000000000..6ca4e2d97 --- /dev/null +++ b/scripts/smoke.mjs @@ -0,0 +1,49 @@ +// Smoke-tests the built artifacts (not src), so a broken published package can't +// slip through. Runs after build. Imports the Node ESM entry and the browser +// ESM entry, checks a reference and a subordinate station resolve their +// prediction data, and asserts the browser bundle contains no node:fs. +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +const dist = new URL("../dist/", import.meta.url); + +function check(db, label) { + const ref = db.stations.find( + (s) => s.type === "reference" && s.harmonic_constituents.length > 0, + ); + assert.ok(ref, `${label}: a reference station with harmonics`); + assert.ok( + ref.harmonic_constituents.length > 0, + `${label}: reference harmonics`, + ); + assert.ok(Object.keys(ref.datums).length > 0, `${label}: reference datums`); + + const sub = db.stations.find((s) => s.type === "subordinate" && s.offsets); + assert.ok(sub, `${label}: a subordinate station`); + assert.ok( + sub.harmonic_constituents.length > 0, + `${label}: subordinate resolves reference harmonics`, + ); + + assert.ok(db.search("seattle")[0]?.name, `${label}: search`); + assert.ok( + db.nearest({ latitude: 47.6, longitude: -122.3 }), + `${label}: nearest`, + ); + assert.ok(db.datums.length > 0, `${label}: datums export`); +} + +check(await import(new URL("node/index.js", dist)), "node ESM"); +check(await import(new URL("browser/index.js", dist)), "browser ESM"); + +const browserSrc = readFileSync( + fileURLToPath(new URL("browser/index.js", dist)), + "utf8", +); +assert.ok( + !/["']node:fs["']|require\(["']fs["']\)/.test(browserSrc), + "browser bundle must not reference node:fs", +); + +console.log("smoke: node ESM + browser ESM OK"); diff --git a/src/search/geo.ts b/src/search/geo.ts index ab044a610..6bb1c19aa 100644 --- a/src/search/geo.ts +++ b/src/search/geo.ts @@ -7,7 +7,8 @@ import KDBush from "kdbush"; * import { createGeoIndex } from "./search-index.js" with { type: "macro" }; */ export async function createGeoIndex() { - const { allStations: stations } = await import("../stations.js"); + const { loadStationMeta } = await import("../station-bundle.js"); + const stations = loadStationMeta(); const index = new KDBush(stations.length); diff --git a/src/search/index.ts b/src/search/index.ts index e1794843f..0a2eda991 100644 --- a/src/search/index.ts +++ b/src/search/index.ts @@ -41,9 +41,23 @@ export type TextSearchOptions = { */ export type StationWithDistance = [Station, number]; -// Load the indexes, which get inlined at build time +// The geo index is small and used by near/nearest/bbox, so load it eagerly. const geoIndex = loadGeoIndex(await createGeoIndex()); -const textIndex = loadTextIndex(await createTextIndex()); + +// The text index costs ~15 MB of heap to build. Consumers that only do geo/id +// lookups (e.g. the signalk-tides plugin on constrained hardware) never call +// search(), so defer building it until the first text search. The serialized +// index string (~1.5 MB) is still inlined at build time; only the expensive +// loadTextIndex build is deferred. +let textIndexData: string | undefined = await createTextIndex(); +let textIndex: ReturnType | undefined; +function getTextIndex() { + if (!textIndex) { + textIndex = loadTextIndex(textIndexData!); + textIndexData = undefined; // free the ~1.5 MB serialized string for GC + } + return textIndex; +} function createFilter( includeAll?: boolean, @@ -122,8 +136,9 @@ export function search( { includeAll, filter, maxResults = 20 }: TextSearchOptions = {}, ): Station[] { const combined = createFilter(includeAll, filter); + const index = getTextIndex(); - const searchOptions: Parameters[1] = {}; + const searchOptions: Parameters[1] = {}; if (combined) { searchOptions.filter = (result) => { @@ -132,7 +147,7 @@ export function search( }; } - const results = textIndex.search(query, searchOptions); + const results = index.search(query, searchOptions); return results .slice(0, maxResults) diff --git a/src/search/text.ts b/src/search/text.ts index 51f6816e0..c2e50f715 100644 --- a/src/search/text.ts +++ b/src/search/text.ts @@ -1,7 +1,9 @@ import MiniSearch, { type Options } from "minisearch"; -import type { Station } from "../types.js"; +import type { StationMeta } from "../types.js"; -const textSearchIndexOptions: Options = { +// Only metadata fields are indexed, so this operates on StationMeta and never +// touches the lazily-loaded harmonics/datums. +const textSearchIndexOptions: Options = { fields: ["name", "region", "country", "continent", "source.id"], extractField: (station, fieldName) => { if (fieldName in station) { @@ -26,14 +28,15 @@ const textSearchIndexOptions: Options = { * import { createTextIndex } from "./text-search-index.js" with { type: "macro" }; */ export async function createTextIndex() { - const { allStations: stations } = await import("../stations.js"); + const { loadStationMeta } = await import("../station-bundle.js"); + const stations = loadStationMeta(); - const index = new MiniSearch(textSearchIndexOptions); + const index = new MiniSearch(textSearchIndexOptions); index.addAll(stations); return JSON.stringify(index.toJSON()); } -export function loadTextIndex(data: string): MiniSearch { - return MiniSearch.loadJSON(data, textSearchIndexOptions); +export function loadTextIndex(data: string): MiniSearch { + return MiniSearch.loadJSON(data, textSearchIndexOptions); } diff --git a/src/station-bundle.ts b/src/station-bundle.ts new file mode 100644 index 000000000..46b49d38e --- /dev/null +++ b/src/station-bundle.ts @@ -0,0 +1,94 @@ +import type { StationData, StationMeta, StationMetaKey } from "./types.js"; + +// Build-time only. This module reads the raw station JSON and splits it into +// light metadata (bundled eagerly) and heavy fields (bundled as unparsed JSON +// strings, parsed one station at a time at runtime). It is never included in +// the runtime bundle: stations.ts imports the create*Json functions as macros +// (only their string results are inlined), and the search index macros call +// loadStationMeta() at build time. + +const META_KEYS: StationMetaKey[] = [ + "name", + "latitude", + "longitude", + "region", + "country", + "continent", + "timezone", + "type", + "disclaimers", + "chart_datum", + "datums_source", + "source", + "license", + "offsets", +]; + +function readAll(): { id: string; data: StationData }[] { + const modules = import.meta.glob("./**/*.json", { + eager: true, + import: "default", + base: "../data", + }); + // Sort explicitly so the metadata array and the geo/text indexes share one + // deterministic station order regardless of the bundler's glob implementation. + return Object.entries(modules) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([path, data]) => ({ + id: path.replace(/^\.\//, "").replace(/\.json$/, ""), + data, + })); +} + +/** + * The set of datum keys present across all stations, computed at build time and + * inlined as a small array literal. Lets consumers (e.g. the API's OpenAPI spec) + * get the datum enum without a runtime scan that would parse every station. + */ +export function createDatumEnum(): string[] { + const datums = new Set(); + for (const { data } of readAll()) { + if (data.datums) + for (const key of Object.keys(data.datums)) datums.add(key); + } + // Sorted for deterministic builds (downstream may embed this, e.g. OpenAPI enums). + return [...datums].sort(); +} + +/** Light metadata for every station, in bundle order. Used at build time. */ +export function loadStationMeta(): StationMeta[] { + return readAll().map(({ id, data }) => { + const meta = { id } as StationMeta; + for (const key of META_KEYS) { + const value = data[key]; + if (value !== undefined) (meta as Record)[key] = value; + } + return meta; + }); +} + +/** + * Metadata array, inlined into the runtime bundle via a macro as an array of + * object literals (the live objects — no retained JSON string, no parse). + */ +export function createStationMeta(): StationMeta[] { + return loadStationMeta(); +} + +/** + * Prediction data (harmonic_constituents, datums, epoch) keyed by station id, + * each a JSON string. Used by the browser build, which inlines this as an object + * of string literals and parses one record on demand. (The Node build reads the + * same records from an off-heap pack file instead — see station-data.ts.) + */ +export function createStationDataById(): Record { + const data: Record = {}; + for (const { id, data: station } of readAll()) { + data[id] = JSON.stringify({ + harmonic_constituents: station.harmonic_constituents ?? [], + datums: station.datums ?? {}, + epoch: station.epoch, + }); + } + return data; +} diff --git a/src/station-data.browser.ts b/src/station-data.browser.ts new file mode 100644 index 000000000..b1be7a903 --- /dev/null +++ b/src/station-data.browser.ts @@ -0,0 +1,16 @@ +// Browser prediction-data source. Selected via the "#station-data" subpath +// import for the browser build (which has no filesystem). The records are +// bundled as JSON strings and parsed one at a time on demand. This costs more +// heap than the Node pack (the strings are on the JS heap), but browsers have no +// equivalent of Node's tight --max-old-space-size limit, and keeping the API +// synchronous matters more than the extra megabytes here. +import { createStationDataById } from "./station-bundle.js" with { type: "macro" }; +import type { PredictionData } from "./types.js"; + +const data: Record = createStationDataById(); + +export function getData(id: string): PredictionData { + const record = data[id]; + if (!record) throw new Error(`No data record for station ${id}`); + return JSON.parse(record); +} diff --git a/src/station-data.ts b/src/station-data.ts new file mode 100644 index 000000000..de3479c75 --- /dev/null +++ b/src/station-data.ts @@ -0,0 +1,31 @@ +// Node prediction-data source. Selected via the "#station-data" subpath import +// for the Node build (and by tests). One file handle is held open; each lookup +// reads only that station's byte range from the pack — the file is never loaded +// whole. So the process holds none of the ~6,000 stations' prediction data (the +// OS page-caches the touched pages, which it can evict under memory pressure), +// and the parsed record is transient. +import { openSync, readSync } from "node:fs"; +import { packIndex } from "./generated/pack-index.js"; +import type { PredictionData } from "./types.js"; + +const fd = openSync(new URL("./generated/stations.pack", import.meta.url), "r"); + +export function getData(id: string): PredictionData { + const range = packIndex[id]; + if (!range) throw new Error(`No data record for station ${id}`); + const [offset, length] = range; + const buffer = Buffer.allocUnsafe(length); + // readSync may return a short read; loop until the whole range is filled so no + // uninitialized bytes from allocUnsafe reach JSON.parse. + let read = 0; + while (read < length) { + const n = readSync(fd, buffer, read, length - read, offset + read); + if (n === 0) { + throw new Error( + `Short read for station ${id}: ${read} of ${length} bytes`, + ); + } + read += n; + } + return JSON.parse(buffer.toString("utf8")); +} diff --git a/src/stations.ts b/src/stations.ts index c67385289..5cf6bd4ff 100644 --- a/src/stations.ts +++ b/src/stations.ts @@ -1,11 +1,54 @@ -import type { Station, StationData } from "./types.js"; +import type { Station, StationMeta } from "./types.js"; +import { createStationMeta } from "./station-bundle.js" with { type: "macro" }; +import { createDatumEnum } from "./station-bundle.js" with { type: "macro" }; +import { getData } from "#station-data"; import quality from "../quality.json" with { type: "json" }; -const modules = import.meta.glob("./**/*.json", { - eager: true, - import: "default", - base: "../data", -}); +/** All datum keys present across the database (e.g. "MLLW", "MSL", "NAVD88"). */ +export const datums: string[] = createDatumEnum(); + +// Metadata (identity + offsets/source/etc) is inlined as object literals. The +// prediction data (harmonic_constituents, datums, epoch) comes from a per-runtime +// source (#station-data): an off-heap pack file on Node, bundled strings in the +// browser. Either way, importing this module holds no station data on the heap — +// a station's record is parsed only when its prediction fields are accessed. +const meta: StationMeta[] = createStationMeta(); + +function makeStation(m: StationMeta): Station { + // Subordinate stations predict from their reference station's harmonics and + // datums (their own offsets still apply); resolve to the reference's record. + const dataId = + m.type === "subordinate" && m.offsets ? m.offsets.reference : m.id; + + const station = { ...m } as Station; + + // Getters keep the sync API: reading these fields parses one station's record. + // No caching — a persistent cache on these module-level objects would pull the + // heavy data back onto the heap. + Object.defineProperties(station, { + harmonic_constituents: { + enumerable: true, + configurable: true, + get: () => getData(dataId).harmonic_constituents, + }, + datums: { + enumerable: true, + configurable: true, + get: () => getData(dataId).datums, + }, + epoch: { + enumerable: true, + configurable: true, + get: () => getData(m.id).epoch, + }, + }); + + return station; +} + +export const allStations: Station[] = meta.map(makeStation); + +export const stationsById = new Map(allStations.map((s) => [s.id, s])); export const qualityMap = new Map(quality.map((s) => [s.id, s])); @@ -13,27 +56,4 @@ export function qualityFilter(station: Station): boolean { return qualityMap.get(station.id)?.accepted ?? false; } -export const allStations: Station[] = Object.entries(modules).map( - ([path, data]) => { - const id = path.replace(/^\.\//, "").replace(/\.json$/, ""); - return { id, ...data }; - }, -); - -export const stationsById = new Map(allStations.map((s) => [s.id, s])); - export const stations: Station[] = allStations.filter(qualityFilter); - -// Populate subordinate stations with datums and harmonic constituents from their reference stations. -allStations.forEach((station) => { - if (station.type === "subordinate") { - const reference = stationsById.get(station.offsets!.reference); - if (!reference) - throw new Error( - `Reference station ${station.offsets!.reference} not found for station ${station.id}`, - ); - - const { datums, harmonic_constituents } = reference; - Object.assign(station, { datums, harmonic_constituents }); - } -}); diff --git a/src/types.ts b/src/types.ts index 0733c7de0..5a2a42c7a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -69,3 +69,33 @@ export interface StationData { export interface Station extends StationData { id: string; } + +// The light fields, bundled eagerly for all stations (~1.5 MB serialized; +// ~15 MB as live objects on the heap). Everything the search/geo/list paths +// need. The prediction data (harmonic_constituents, datums, epoch) is loaded +// per station from the pack — see station-data.ts. +export type StationMetaKey = + | "name" + | "latitude" + | "longitude" + | "region" + | "country" + | "continent" + | "timezone" + | "type" + | "disclaimers" + | "chart_datum" + | "datums_source" + | "source" + | "license" + | "offsets"; + +export type StationMeta = { id: string } & Pick; + +// The lazily-loaded prediction data, resolved per station from the data source +// (an off-heap pack file on Node, bundled JSON strings in the browser). +export interface PredictionData { + harmonic_constituents: HarmonicConstituent[]; + datums: Record; + epoch?: StationData["epoch"]; +} diff --git a/test/station-bundle.test.ts b/test/station-bundle.test.ts new file mode 100644 index 000000000..66f10c1ee --- /dev/null +++ b/test/station-bundle.test.ts @@ -0,0 +1,33 @@ +import { describe, test, expect } from "vitest"; +import { datums, stations, stationsById, allStations } from "../src/index.js"; + +describe("datums export", () => { + test("is the set of datum keys present in the database", () => { + expect(datums).toContain("MLLW"); + expect(datums).toContain("MSL"); + expect(datums.length).toBeGreaterThan(10); + + // Every datum key on a reference station is covered by the export. + const ref = stations.find( + (s) => s.type === "reference" && Object.keys(s.datums).length > 0, + )!; + for (const key of Object.keys(ref.datums)) expect(datums).toContain(key); + }); +}); + +describe("lazily loaded station data", () => { + test("reference stations resolve their own harmonics and datums", () => { + const ref = stations.find( + (s) => s.type === "reference" && s.harmonic_constituents.length > 0, + )!; + expect(ref.harmonic_constituents[0]).toHaveProperty("amplitude"); + expect(Object.keys(ref.datums).length).toBeGreaterThan(0); + }); + + test("subordinate stations inherit harmonics and datums from their reference", () => { + const sub = allStations.find((s) => s.type === "subordinate" && s.offsets)!; + const ref = stationsById.get(sub.offsets!.reference)!; + expect(sub.harmonic_constituents).toEqual(ref.harmonic_constituents); + expect(sub.datums).toEqual(ref.datums); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index d3e5c1f73..7207f4136 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ "compilerOptions": { "rootDir": "./src", "outDir": "./dist", - "types": ["vite/client"], + "types": ["vite/client", "node"], "declarationMap": true, "tsBuildInfoFile": "./dist/.tsbuildinfo" }, diff --git a/tsdown.config.ts b/tsdown.config.ts index 0392b66d6..8307a2953 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -1,13 +1,22 @@ import { defineConfig } from "tsdown"; import macros from "unplugin-macros/rolldown"; -export default defineConfig({ - exports: true, +// Two builds. The Node build resolves `#station-data` to station-data.ts (reads +// the off-heap pack file); the browser build resolves it to +// station-data.browser.ts (bundled JSON strings). package.json `exports` +// conditions select between dist/node and dist/browser. +const base = { + entry: ["./src/index.ts"], dts: true, minify: true, - format: ["cjs", "esm"], sourcemap: true, target: "es2020", - platform: "neutral", plugins: [macros()], -}); +}; + +// ESM only. kdbush and geokdbush are ESM-only, so a CJS build can't require them +// cleanly; all first-party consumers use ESM. `#station-data` resolves per build. +export default defineConfig([ + { ...base, platform: "neutral", format: ["esm"], outDir: "dist/node" }, + { ...base, platform: "browser", format: ["esm"], outDir: "dist/browser" }, +]);