From 81e0e25937bacf5f1ae50f6d224916ab44155c21 Mon Sep 17 00:00:00 2001 From: Brandon Keepers Date: Wed, 15 Jul 2026 08:37:57 -0400 Subject: [PATCH 1/4] Load station harmonics/datums lazily to cut memory ~40% MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Importing @neaps/tide-database eagerly parsed all 6,000+ stations into live JS objects, costing ~118 MB of heap / ~660 MB RSS. This OOMs memory-constrained consumers (signalk-tides on a Victron Cerbo GX, openwatersio/signalk-tides#103) and dominates the tides API's serverless cold-start CPU. Split each station into light metadata (bundled eagerly as object literals, ~15 MB) and heavy fields — harmonic_constituents, datums, epoch — bundled as unparsed per-station JSON strings and parsed on demand via getters. Reading a station's harmonics parses just that one station; subordinate stations resolve to their reference's data. The ~15 MB MiniSearch text index is also deferred until the first search(). The public API stays synchronous. Adds a build-time `datums` export so consumers (the API's OpenAPI spec) can get the datum enum without a runtime scan that would re-parse every station. Import + one nearest() lookup: heap 118 MB -> 69 MB (84 MB once text search is used). All 31,862 tests pass. See docs/lazy-loading.md. Refs openwatersio/signalk-tides#103 --- docs/lazy-loading.md | 250 ++++++++++++++++++++++++++++++++++++ src/search/geo.ts | 3 +- src/search/index.ts | 19 ++- src/search/text.ts | 15 ++- src/station-bundle.ts | 89 +++++++++++++ src/stations.ts | 98 +++++++++----- src/types.ts | 21 +++ test/station-bundle.test.ts | 33 +++++ 8 files changed, 488 insertions(+), 40 deletions(-) create mode 100644 docs/lazy-loading.md create mode 100644 src/station-bundle.ts create mode 100644 test/station-bundle.test.ts diff --git a/docs/lazy-loading.md b/docs/lazy-loading.md new file mode 100644 index 000000000..4603612f6 --- /dev/null +++ b/docs/lazy-loading.md @@ -0,0 +1,250 @@ +# Lazy-loading proposal + +Status: **Phase 0 implemented** (see the interim section below); phases 1–5 are +a proposal for discussion. + +## Problem + +[`src/stations.ts`](../src/stations.ts) uses `import.meta.glob(..., { eager: true })` +to inline **every** station's JSON into the bundle at build time. So +`dist/index.js` is ~23 MB (32.7 MB raw across 8,290 files), and _any_ import of +the package realizes all of it — even a health check or a single-station lookup. + +Measured cost — two symptoms, one cause: + +- **CPU:** importing `@neaps/tide-database` is ~358 ms; a warm prediction is + ~2 ms. On a platform that cold-starts frequently (the tides API burns ~95% of + its CPU budget), that module evaluation _is_ the bill. Caching only helps the + head of the request distribution; the long tail keeps paying the cold start. +- **Memory:** parsing all 6,085 stations into live JS objects costs **118 MB of + heap / 663 MB RSS** (the predictor is 4 MB — the database is all of it). This + OOMs memory-constrained devices: signalk-tides + ([#103](https://github.com/openwatersio/signalk-tides/issues/103)) crashes a + Victron Cerbo GX with a V8 "Reached heap limit" error, because the 118 MB + baseline consumes the constrained heap's headroom. The prediction hot loop + itself does **not** leak — measured flat over 600 iterations (~10 h of plugin + runtime). + +The fix is to stop loading 8,290 stations to answer a request about one. + +## Key insight: metadata is 4.6% of the data + +| | Size | Loaded | +| ----------------------------------------------------------------------------------------------------- | ----------- | ----------------------- | +| Metadata (`id, name, lat, lon, region, country, continent, timezone, type`, + `ref` for subordinates) | **1.49 MB** | eagerly, bundled | +| Harmonics/datums/offsets/epoch/source/license (the heavy part) | 31.2 MB | **lazily, per station** | + +Everything the search/geo/list endpoints need is in the metadata. Only actual +_prediction_ needs the heavy part, and only for the one (or two) stations +involved. The geo (KDBush) and text (MiniSearch) indexes are already built at +build time and inlined as compact base64 — they stay as-is. + +## Data model + +Split the station type in two: + +```ts +// Bundled for all 8,290 stations (~1.5 MB). Powers search, near, bbox, list, +// and station summaries. +export interface StationMeta { + id: string; + name: string; + latitude: number; + longitude: number; + region?: string; + country: string; + continent: string; + timezone: string; + type: "reference" | "subordinate"; + ref?: string; // reference station id, for subordinates +} + +// The heavy part, loaded on demand. +export interface StationData { + /* harmonic_constituents, datums, offsets, ... */ +} + +// Meta + data, the shape prediction needs. What today's `Station` is. +export type Station = StationMeta & StationData; +``` + +## Pluggable data source + +The heavy data lives somewhere different per runtime, so make the source an +injectable interface with a small set of built-ins: + +```ts +export interface StationDataSource { + get(id: string): Promise; +} +``` + +| Source | Runtime | Backing | +| --------------------- | ----------------------------- | ------------------------------------------------------------------------------------------ | +| `bundledSource()` | anywhere (small/offline) | current eager glob — opt-in, pays the 23 MB | +| `fsSource(dir)` | Node (Vercel, `neaps-server`) | reads `data//.json` off disk; the npm package already ships `data/` | +| `httpSource(baseUrl)` | edge / browser | `fetch(`${base}/${id}.json`)` from R2 or a release asset — mirrors the gebco tiles pattern | +| `r2Source(bucket)` | Cloudflare Workers | `bucket.get(`${id}.json`)` binding | + +The metadata index is always bundled, so `search`/`near`/`bbox`/list stay +synchronous and instant on every runtime. Only heavy loads are async. + +## Code sketch + +`stations.ts` — bundle metadata, not full stations: + +```ts +// Build-time: emit meta only (see "Build" below). Eager, but ~1.5 MB not 23 MB. +import metaList from "../data/stations.meta.json" with { type: "json" }; + +export const stationsMeta: StationMeta[] = metaList; +export const metaById = new Map(stationsMeta.map((m) => [m.id, m])); +``` + +A loader that resolves the subordinate→reference wrinkle (2,239 of 8,290 +stations borrow their reference's constituents): + +```ts +export function createLoader(source: StationDataSource) { + const cache = new Map>(); + + function load(id: string): Promise { + let pending = cache.get(id); + if (!pending) { + pending = resolve(id); + cache.set(id, pending); + } + return pending; + } + + async function resolve(id: string): Promise { + const meta = metaById.get(id); + if (!meta) throw new Error(`Station ${id} not found`); + const data = await source.get(id); + // Subordinate stations predict from their reference's harmonics/datums. + if (meta.type === "subordinate" && meta.ref) { + const ref = await load(meta.ref); + return { + ...meta, + ...data, + datums: ref.datums, + harmonic_constituents: ref.harmonic_constituents, + }; + } + return { ...meta, ...data }; + } + + return { load }; +} +``` + +`near`/`search`/`bbox` don't change except their element type becomes +`StationMeta` — they already only touch light fields (they map index ids to +positions/names, never harmonics). + +## Consumer ripple + +Resolving a station by id becomes **async**. That's the one real cost. + +- `neaps`: `findStation(id)` and the coordinate-based prediction entry points + become `async` (they must load harmonics before predicting). Prediction math + itself stays sync — `station.getTimelinePrediction(...)` is unchanged once you + hold a loaded `Station`. +- `@neaps/api`: route handlers already run in Express and can `await`. The + handlers become `res.json(await station.getTimelinePrediction(...))`. +- `openapi.ts`: today it does `stations.flatMap(...)` at module top level just to + build the datum enum — that alone forces the full parse. Switch to the fixed + oceanographic datum list (or derive from metadata), so importing the API no + longer touches heavy data. + +## Build + +`tsdown`/vite step emits two things instead of one eager glob: + +1. `data/stations.meta.json` — the 1.5 MB metadata array (bundled). +2. The per-station heavy JSON — already exists as `data//.json`; + ship it in the npm package (`fsSource`) and/or publish to R2 / a release + asset (`httpSource`/`r2Source`), stable-named per release like PR #92 does + for the tileset. + +## Versioning + +This changes the shape and sync-ness of the public API, so it's a **major** +bump. Migration aids: + +- Keep `bundledSource()` so existing offline/sync consumers can opt back into the + old all-in-memory behavior with one line. +- Export `stationsMeta` (sync, light) as the replacement for most `stations` + uses (search results, lists, maps) — those never needed harmonics. + +## Relationship to the vector tileset (PR #92) + +Complementary, not competing. The tileset offloads the **map/search/near** +consumer to the client (rendered from a PMTiles file in R2, zero API CPU). This +proposal fixes the **prediction** path (`/:id/timeline`), which is id-keyed and +can't be served from geo-indexed tiles. Do this first — it removes ~95% of the +API's cold-start cost; ship #92 when you want the map to stop hitting the API at +all. + +## Phase 0 (interim): defer parsing, keep the sync API + +The full proposal makes station resolution async, which ripples through `neaps` +and `@neaps/api`. That's too slow for the OOM. A smaller change lands first and +fixes the memory crisis without touching the sync contract: + +Bundle the heavy data as **unparsed JSON strings** instead of live objects, and +parse one station on first access: + +```ts +// Build emits stations.meta.json (parsed eagerly, ~15-20 MB heap) and a +// data map of id -> raw JSON string (kept as strings, ~23 MB, never parsed +// until touched). Vite: import.meta.glob("./**/*.json", { query: "?raw", ... }). +import dataStrings from "../data/stations.data.js"; // { [id]: string } + +function attachLazyData(meta: StationMeta): Station { + let parsed: StationData | undefined; + const load = () => (parsed ??= JSON.parse(dataStrings[meta.id])); + return Object.defineProperties( + { ...meta }, + { + harmonic_constituents: { + get: () => resolveHarmonics(meta, load), + enumerable: true, + }, + datums: { get: () => resolveDatums(meta, load), enumerable: true }, + }, + ); +} +``` + +`useStation` in the predictor destructures `{ datums, harmonic_constituents }`, +so the getters fire for exactly the one station being predicted — nothing else +parses. `near`/`bbox` read only metadata and never trigger a parse. The +~15 MB MiniSearch text index is also deferred until the first `search()` call, +which geo/id-only consumers (like the plugin) never make. + +Measured result (import + one `nearest()`, GC'd): heap **118 MB → 69 MB** +(84 MB once text search is used), all 31,859 tests pass, fully synchronous, a +tide-database-only change (+ rebuild + republish). The heavy data still ships +in the bundle as ~17 MB of strings, so this is the floor for a bundled+offline +database; the async pluggable-source version below is what gets it to ~15 MB and +shrinks the edge bundle. RSS stays high on a machine with abundant RAM (V8 keeps +its peak reservation), but the reported crash is a V8 _heap-limit_ OOM, and +under a configured `--max-old-space-size` the process now stays well within it. + +A companion `@neaps/api` change is required: `openapi.ts` builds its datum enum +with `stations.flatMap((s) => Object.keys(s.datums))`, which touches every +station and re-triggers a full parse. tide-database now exports a build-time +`datums` constant for it to import instead. + +## Rollout order + +0. **(interim, urgent)** Defer parsing — heavy data bundled as strings, parsed + per station on access. Sync API unchanged. Fixes the OOM. +1. Build emits `stations.meta.json` + keeps per-station files; add + `stationsMeta` export alongside the existing `stations` (no breakage yet). +2. Add `StationDataSource` + `createLoader`; `fsSource` for Node, `httpSource` + for edge. +3. Make `neaps` station resolution async; update `@neaps/api` handlers to await. +4. Point the tides API's Worker at `httpSource`/`r2Source`; drop `bundledSource`. +5. Deprecate the eager `stations` export (major bump). diff --git a/src/search/geo.ts b/src/search/geo.ts index ab044a610..6bb1c19aa 100644 --- a/src/search/geo.ts +++ b/src/search/geo.ts @@ -7,7 +7,8 @@ import KDBush from "kdbush"; * import { createGeoIndex } from "./search-index.js" with { type: "macro" }; */ export async function createGeoIndex() { - const { allStations: stations } = await import("../stations.js"); + const { loadStationMeta } = await import("../station-bundle.js"); + const stations = loadStationMeta(); const index = new KDBush(stations.length); diff --git a/src/search/index.ts b/src/search/index.ts index e1794843f..33398a2fc 100644 --- a/src/search/index.ts +++ b/src/search/index.ts @@ -41,9 +41,19 @@ export type TextSearchOptions = { */ export type StationWithDistance = [Station, number]; -// Load the indexes, which get inlined at build time +// The geo index is small and used by near/nearest/bbox, so load it eagerly. const geoIndex = loadGeoIndex(await createGeoIndex()); -const textIndex = loadTextIndex(await createTextIndex()); + +// The text index costs ~15 MB of heap to build. Consumers that only do geo/id +// lookups (e.g. the signalk-tides plugin on constrained hardware) never call +// search(), so defer building it until the first text search. The serialized +// index string (~1.5 MB) is still inlined at build time; only the expensive +// loadTextIndex build is deferred. +const textIndexData = await createTextIndex(); +let textIndex: ReturnType | undefined; +function getTextIndex() { + return (textIndex ??= loadTextIndex(textIndexData)); +} function createFilter( includeAll?: boolean, @@ -122,8 +132,9 @@ export function search( { includeAll, filter, maxResults = 20 }: TextSearchOptions = {}, ): Station[] { const combined = createFilter(includeAll, filter); + const index = getTextIndex(); - const searchOptions: Parameters[1] = {}; + const searchOptions: Parameters[1] = {}; if (combined) { searchOptions.filter = (result) => { @@ -132,7 +143,7 @@ export function search( }; } - const results = textIndex.search(query, searchOptions); + const results = index.search(query, searchOptions); return results .slice(0, maxResults) diff --git a/src/search/text.ts b/src/search/text.ts index 51f6816e0..c2e50f715 100644 --- a/src/search/text.ts +++ b/src/search/text.ts @@ -1,7 +1,9 @@ import MiniSearch, { type Options } from "minisearch"; -import type { Station } from "../types.js"; +import type { StationMeta } from "../types.js"; -const textSearchIndexOptions: Options = { +// Only metadata fields are indexed, so this operates on StationMeta and never +// touches the lazily-loaded harmonics/datums. +const textSearchIndexOptions: Options = { fields: ["name", "region", "country", "continent", "source.id"], extractField: (station, fieldName) => { if (fieldName in station) { @@ -26,14 +28,15 @@ const textSearchIndexOptions: Options = { * import { createTextIndex } from "./text-search-index.js" with { type: "macro" }; */ export async function createTextIndex() { - const { allStations: stations } = await import("../stations.js"); + const { loadStationMeta } = await import("../station-bundle.js"); + const stations = loadStationMeta(); - const index = new MiniSearch(textSearchIndexOptions); + const index = new MiniSearch(textSearchIndexOptions); index.addAll(stations); return JSON.stringify(index.toJSON()); } -export function loadTextIndex(data: string): MiniSearch { - return MiniSearch.loadJSON(data, textSearchIndexOptions); +export function loadTextIndex(data: string): MiniSearch { + return MiniSearch.loadJSON(data, textSearchIndexOptions); } diff --git a/src/station-bundle.ts b/src/station-bundle.ts new file mode 100644 index 000000000..a6f553a4c --- /dev/null +++ b/src/station-bundle.ts @@ -0,0 +1,89 @@ +import type { StationData, StationMeta, StationMetaKey } from "./types.js"; + +// Build-time only. This module reads the raw station JSON and splits it into +// light metadata (bundled eagerly) and heavy fields (bundled as unparsed JSON +// strings, parsed one station at a time at runtime). It is never included in +// the runtime bundle: stations.ts imports the create*Json functions as macros +// (only their string results are inlined), and the search index macros call +// loadStationMeta() at build time. + +const META_KEYS: StationMetaKey[] = [ + "name", + "latitude", + "longitude", + "region", + "country", + "continent", + "timezone", + "type", + "disclaimers", + "chart_datum", + "datums_source", + "source", + "license", + "offsets", +]; + +function readAll(): { id: string; data: StationData }[] { + const modules = import.meta.glob("./**/*.json", { + eager: true, + import: "default", + base: "../data", + }); + // Object.entries preserves the glob's sorted key order, so the metadata array, + // the heavy array, and the geo/text indexes all share one station ordering. + return Object.entries(modules).map(([path, data]) => ({ + id: path.replace(/^\.\//, "").replace(/\.json$/, ""), + data, + })); +} + +/** + * The set of datum keys present across all stations, computed at build time and + * inlined as a small array literal. Lets consumers (e.g. the API's OpenAPI spec) + * get the datum enum without a runtime scan that would parse every station. + */ +export function createDatumEnum(): string[] { + const datums = new Set(); + for (const { data } of readAll()) { + if (data.datums) + for (const key of Object.keys(data.datums)) datums.add(key); + } + return [...datums]; +} + +/** Light metadata for every station, in bundle order. Used at build time. */ +export function loadStationMeta(): StationMeta[] { + return readAll().map(({ id, data }) => { + const meta = { id } as StationMeta; + for (const key of META_KEYS) { + const value = data[key]; + if (value !== undefined) (meta as Record)[key] = value; + } + return meta; + }); +} + +/** + * Metadata array, inlined into the runtime bundle via a macro as an array of + * object literals (the live objects — no retained JSON string, no parse). + */ +export function createStationMeta(): StationMeta[] { + return loadStationMeta(); +} + +/** + * Heavy fields (harmonic_constituents, datums, epoch) as an array of per-station + * JSON strings, inlined as an array of string literals. The strings are the live + * data; each is JSON.parsed on demand, so importing the bundle never + * materializes all stations' harmonics at once. + */ +export function createStationHeavy(): string[] { + return readAll().map(({ data }) => + JSON.stringify({ + harmonic_constituents: data.harmonic_constituents ?? [], + datums: data.datums ?? {}, + epoch: data.epoch, + }), + ); +} diff --git a/src/stations.ts b/src/stations.ts index c67385289..f2700528c 100644 --- a/src/stations.ts +++ b/src/stations.ts @@ -1,39 +1,79 @@ -import type { Station, StationData } from "./types.js"; +import type { + HarmonicConstituent, + Station, + StationData, + StationMeta, +} from "./types.js"; +import { createStationMeta } from "./station-bundle.js" with { type: "macro" }; +import { createStationHeavy } from "./station-bundle.js" with { type: "macro" }; +import { createDatumEnum } from "./station-bundle.js" with { type: "macro" }; import quality from "../quality.json" with { type: "json" }; -const modules = import.meta.glob("./**/*.json", { - eager: true, - import: "default", - base: "../data", -}); +/** All datum keys present across the database (e.g. "MLLW", "MSL", "NAVD88"). */ +export const datums: string[] = createDatumEnum(); -export const qualityMap = new Map(quality.map((s) => [s.id, s])); +// Metadata is inlined as object literals (~15 MB of live objects). Heavy fields +// are inlined as an array of per-station JSON string literals and parsed on +// demand, so importing this module no longer materializes all 6,000+ stations' +// harmonics (which cost ~118 MB of heap / ~660 MB RSS eagerly). +const meta: StationMeta[] = createStationMeta(); +const heavy: string[] = createStationHeavy(); -export function qualityFilter(station: Station): boolean { - return qualityMap.get(station.id)?.accepted ?? false; +const indexById = new Map(meta.map((m, i) => [m.id, i] as const)); + +interface HeavyFields { + harmonic_constituents: HarmonicConstituent[]; + datums: Record; + epoch?: StationData["epoch"]; +} + +function parseHeavy(index: number): HeavyFields { + return JSON.parse(heavy[index]!); +} + +function makeStation(m: StationMeta, index: number): Station { + // Subordinate stations predict from their reference station's harmonics and + // datums (their own offsets still apply). Resolve to the reference's heavy + // data; fall back to self if the reference is somehow missing. + const dataIndex = + m.type === "subordinate" && m.offsets + ? (indexById.get(m.offsets.reference) ?? index) + : index; + + const station = { ...m } as Station; + + // Getters keep the sync API: reading these fields parses one station's heavy + // blob. No caching — a persistent cache on these module-level objects would + // grow back toward the full 118 MB on a process that touches every station. + Object.defineProperties(station, { + harmonic_constituents: { + enumerable: true, + configurable: true, + get: () => parseHeavy(dataIndex).harmonic_constituents, + }, + datums: { + enumerable: true, + configurable: true, + get: () => parseHeavy(dataIndex).datums, + }, + epoch: { + enumerable: true, + configurable: true, + get: () => parseHeavy(index).epoch, + }, + }); + + return station; } -export const allStations: Station[] = Object.entries(modules).map( - ([path, data]) => { - const id = path.replace(/^\.\//, "").replace(/\.json$/, ""); - return { id, ...data }; - }, -); +export const allStations: Station[] = meta.map(makeStation); export const stationsById = new Map(allStations.map((s) => [s.id, s])); -export const stations: Station[] = allStations.filter(qualityFilter); +export const qualityMap = new Map(quality.map((s) => [s.id, s])); -// Populate subordinate stations with datums and harmonic constituents from their reference stations. -allStations.forEach((station) => { - if (station.type === "subordinate") { - const reference = stationsById.get(station.offsets!.reference); - if (!reference) - throw new Error( - `Reference station ${station.offsets!.reference} not found for station ${station.id}`, - ); - - const { datums, harmonic_constituents } = reference; - Object.assign(station, { datums, harmonic_constituents }); - } -}); +export function qualityFilter(station: Station): boolean { + return qualityMap.get(station.id)?.accepted ?? false; +} + +export const stations: Station[] = allStations.filter(qualityFilter); diff --git a/src/types.ts b/src/types.ts index 0733c7de0..b7dbbf8ee 100644 --- a/src/types.ts +++ b/src/types.ts @@ -69,3 +69,24 @@ export interface StationData { export interface Station extends StationData { id: string; } + +// The light fields, bundled eagerly for all stations (~1.5 MB). Everything the +// search/geo/list paths need. The heavy fields (harmonic_constituents, datums, +// epoch) are loaded lazily per station — see station-bundle.ts. +export type StationMetaKey = + | "name" + | "latitude" + | "longitude" + | "region" + | "country" + | "continent" + | "timezone" + | "type" + | "disclaimers" + | "chart_datum" + | "datums_source" + | "source" + | "license" + | "offsets"; + +export type StationMeta = { id: string } & Pick; diff --git a/test/station-bundle.test.ts b/test/station-bundle.test.ts new file mode 100644 index 000000000..66f10c1ee --- /dev/null +++ b/test/station-bundle.test.ts @@ -0,0 +1,33 @@ +import { describe, test, expect } from "vitest"; +import { datums, stations, stationsById, allStations } from "../src/index.js"; + +describe("datums export", () => { + test("is the set of datum keys present in the database", () => { + expect(datums).toContain("MLLW"); + expect(datums).toContain("MSL"); + expect(datums.length).toBeGreaterThan(10); + + // Every datum key on a reference station is covered by the export. + const ref = stations.find( + (s) => s.type === "reference" && Object.keys(s.datums).length > 0, + )!; + for (const key of Object.keys(ref.datums)) expect(datums).toContain(key); + }); +}); + +describe("lazily loaded station data", () => { + test("reference stations resolve their own harmonics and datums", () => { + const ref = stations.find( + (s) => s.type === "reference" && s.harmonic_constituents.length > 0, + )!; + expect(ref.harmonic_constituents[0]).toHaveProperty("amplitude"); + expect(Object.keys(ref.datums).length).toBeGreaterThan(0); + }); + + test("subordinate stations inherit harmonics and datums from their reference", () => { + const sub = allStations.find((s) => s.type === "subordinate" && s.offsets)!; + const ref = stationsById.get(sub.offsets!.reference)!; + expect(sub.harmonic_constituents).toEqual(ref.harmonic_constituents); + expect(sub.datums).toEqual(ref.datums); + }); +}); From 991ad4f2eeb711723c0bf7a5f8d1f5b4541cfe2c Mon Sep 17 00:00:00 2001 From: Brandon Keepers Date: Wed, 15 Jul 2026 10:22:55 -0400 Subject: [PATCH 2/4] Revise lazy-loading plan: additive /async entry, slim/pack tiers - Frame the async work as a non-breaking @neaps/tide-database/async subpath (main entry unchanged), not a major bump. - Two tiers: slim metadata (identity, bundled) + heavy pack (per station). - Derive source.id from the id (/, verified across all 8,290 stations) instead of storing it; build asserts the invariant. - Bundle the byte-range index (no separate .idx file) so the client is dependency-free. - Add a geo/text index section (geo ~66 KB eager; text ~15 MB lazy or linear-scan) and the expected ~10-15 MB pack baseline. - Reconcile: openapi datums export is done in Phase 0; fix station/test counts. --- docs/lazy-loading.md | 364 +++++++++++++++++++++++++++++-------------- 1 file changed, 248 insertions(+), 116 deletions(-) diff --git a/docs/lazy-loading.md b/docs/lazy-loading.md index 4603612f6..ef4564547 100644 --- a/docs/lazy-loading.md +++ b/docs/lazy-loading.md @@ -1,7 +1,10 @@ # Lazy-loading proposal -Status: **Phase 0 implemented** (see the interim section below); phases 1–5 are -a proposal for discussion. +Status: **Phase 0 implemented** (see the interim section below). The proposed +next step is a new, **additive** `@neaps/tide-database/async` entry point that +keeps heavy station data out of the JavaScript bundle and in a single indexed +pack file. The existing `@neaps/tide-database` entry is unchanged — this is not a +breaking change. ## Problem @@ -16,8 +19,9 @@ Measured cost — two symptoms, one cause: ~2 ms. On a platform that cold-starts frequently (the tides API burns ~95% of its CPU budget), that module evaluation _is_ the bill. Caching only helps the head of the request distribution; the long tail keeps paying the cold start. -- **Memory:** parsing all 6,085 stations into live JS objects costs **118 MB of - heap / 663 MB RSS** (the predictor is 4 MB — the database is all of it). This +- **Memory:** parsing all 8,290 stations (filtered to ~6,177 quality) into live + JS objects costs **118 MB of heap / 663 MB RSS** (the predictor is 4 MB — the + database is all of it). This OOMs memory-constrained devices: signalk-tides ([#103](https://github.com/openwatersio/signalk-tides/issues/103)) crashes a Victron Cerbo GX with a V8 "Reached heap limit" error, because the 118 MB @@ -27,27 +31,36 @@ Measured cost — two symptoms, one cause: The fix is to stop loading 8,290 stations to answer a request about one. -## Key insight: metadata is 4.6% of the data +## Key insight: identity is ~3% of the data -| | Size | Loaded | -| ----------------------------------------------------------------------------------------------------- | ----------- | ----------------------- | -| Metadata (`id, name, lat, lon, region, country, continent, timezone, type`, + `ref` for subordinates) | **1.49 MB** | eagerly, bundled | -| Harmonics/datums/offsets/epoch/source/license (the heavy part) | 31.2 MB | **lazily, per station** | +Split every station into two tiers: -Everything the search/geo/list endpoints need is in the metadata. Only actual -_prediction_ needs the heavy part, and only for the one (or two) stations -involved. The geo (KDBush) and text (MiniSearch) indexes are already built at -build time and inlined as compact base64 — they stay as-is. +| Tier | Fields | Size | Loaded | +| ----------------- | ------------------------------------------------------------------------------ | ----------- | ----------------- | +| **Slim metadata** | id, name, lat, lon, region, country, continent, timezone, type | **~0.9 MB** | bundled, eager | +| **Heavy pack** | harmonic_constituents, datums, offsets, epoch, source, license, chart_datum, … | ~32 MB | pack, per station | -## Data model +Slim metadata is everything needed to _summarize_ a station in a search result, +nearby list, or map pin. The heavy pack is what you need to _predict_ at a +station or show its full detail — loaded only when the user opens one. (Does the +user need the source license to decide whether to view a station? No — so it +lives in the pack.) + +`source.id` is **not** stored: every station id is `/` (e.g. +`noaa/8722588`), so it is `id.slice(id.indexOf("/") + 1)`. Verified across all +8,290 stations — every id has that shape and the derived source-id equals the +record's `source.id`. So `findStation` by source id and the text index's +`source.id` field both derive it from the id, with no heavy load and no duplicate +field. The build asserts the invariant. See "Search indexes" below for how the +geo and text indexes fit in. -Split the station type in two: +## Data model ```ts -// Bundled for all 8,290 stations (~1.5 MB). Powers search, near, bbox, list, -// and station summaries. -export interface StationMeta { - id: string; +// Bundled for all ~8,290 stations (~0.9 MB), parsed once. Everything the +// search / near / bbox / list paths need to render a summary. +export interface StationSummary { + id: string; // "/" name: string; latitude: number; longitude: number; @@ -56,22 +69,55 @@ export interface StationMeta { continent: string; timezone: string; type: "reference" | "subordinate"; - ref?: string; // reference station id, for subordinates } -// The heavy part, loaded on demand. +// Loaded from the pack on demand: the full station record. export interface StationData { - /* harmonic_constituents, datums, offsets, ... */ + harmonic_constituents: HarmonicConstituent[]; + datums: Record; + offsets?: { reference: string; height: unknown; time: unknown }; + epoch?: { start: string; end: string }; + source: { id: string; name: string; url: string }; + license: unknown; + disclaimers: string; + chart_datum: string; } -// Meta + data, the shape prediction needs. What today's `Station` is. -export type Station = StationMeta & StationData; +// Summary + data — what prediction and detail views need. Today's `Station`. +export type Station = StationSummary & StationData; +``` + +The async entry exposes exactly these two halves: `stationsMeta: StationSummary[]` +(bundled) and `getStation(id): Promise` (loads the pack record). A +consumer that wants the full metadata — license, source, epoch — loads the pack; +a consumer that only lists or maps stations never does. + +## Proposed storage: an indexed pack + +Ship one `stations.pack` — concatenated UTF-8 JSON records — plus a compact +index (station id → byte offset + length) that is **bundled into the JS**, not +shipped as a separate file: + +```text +stations.pack # concatenated UTF-8 JSON records (npm asset / R2) +index: Record # bundled in the JS entry, ~0.3 MB ``` -## Pluggable data source +Bundling the index is what keeps the client dependency-free: there is no `.idx` +format to parse, only `pack[offset .. offset+length]` → `JSON.parse`. For example +the index records that `noaa/9414290` occupies a particular byte range in +`stations.pack`; loading it reads and parses only that range. This avoids both +thousands of installed files and keeping the complete heavy dataset in the JS +heap. (The index and pack are produced by the same build and versioned together, +so they cannot drift — see "Build".) + +Do not gzip the complete pack as one stream: retrieving a record near the end +would require decompressing everything before it. Initially, leave the pack +uncompressed. If transfer size later proves important, compress records +individually or divide the file into independently compressed blocks. -The heavy data lives somewhere different per runtime, so make the source an -injectable interface with a small set of built-ins: +The heavy data lives behind a small injectable interface so the same loader can +work in different runtimes: ```ts export interface StationDataSource { @@ -79,103 +125,188 @@ export interface StationDataSource { } ``` -| Source | Runtime | Backing | -| --------------------- | ----------------------------- | ------------------------------------------------------------------------------------------ | -| `bundledSource()` | anywhere (small/offline) | current eager glob — opt-in, pays the 23 MB | -| `fsSource(dir)` | Node (Vercel, `neaps-server`) | reads `data//.json` off disk; the npm package already ships `data/` | -| `httpSource(baseUrl)` | edge / browser | `fetch(`${base}/${id}.json`)` from R2 or a release asset — mirrors the gebco tiles pattern | -| `r2Source(bucket)` | Cloudflare Workers | `bucket.get(`${id}.json`)` binding | +| Source | Runtime | Backing | +| ----------------------- | ----------------------- | --------------------------------------------------------- | +| `packFileSource(path)` | Node | `fs.read()` of the indexed byte range | +| `packHttpSource(url)` | browser / edge | HTTP Range request for the indexed byte range | +| `packObjectSource(obj)` | Cloudflare Workers | R2 range read | +| `bundledSource()` | anywhere, compatibility | current inlined strings; opt-in and retains the full data | + +For HTTP, the server must support byte ranges and return a stable versioned +pack. Browser bundlers have no universal way to serve an npm package asset, so +browser/edge consumers supply the pack URL (an R2 or release-asset URL). + +Slim metadata and the index are always bundled, so `search`/`near`/`bbox`/list +stay synchronous and instant on every runtime. Only heavy loads are async. + +## Search indexes (geo + text) + +The two indexes are very different sizes and handled differently: + +| Index | Serialized | Built in memory | Used by | Plan | +| ----------------- | ---------- | --------------- | --------------------- | ------------------------------ | +| geo (KDBush) | ~66 KB | ~66 KB | near / nearest / bbox | bundle, eager — negligible | +| text (MiniSearch) | ~1.5 MB | **~15 MB** | `search()` only | build on first search, or skip | -The metadata index is always bundled, so `search`/`near`/`bbox`/list stay -synchronous and instant on every runtime. Only heavy loads are async. +`near`/`nearest`/`bbox` resolve coordinates → station ids from the ~66 KB geo +index with zero heavy loading; you then `getStation` only the one(s) you want. + +The text index is the single largest optional cost. A search touches every +station, so it can't be range-read from the pack — it's all-or-nothing. +Therefore it is **built lazily on the first `search()`** (Phase 0 already does +this), and geo/id-only consumers like the plugin never pay it. If even the lazy +15 MB is unwanted, a memory-critical build can drop MiniSearch entirely and do a +linear fuzzy scan over the bundled slim-metadata `name` field — 8,290 names is +small enough to scan interactively at zero extra memory. Its `source.id` field is +derived from the id. ## Code sketch -`stations.ts` — bundle metadata, not full stations: +Slim metadata, bundled: ```ts -// Build-time: emit meta only (see "Build" below). Eager, but ~1.5 MB not 23 MB. -import metaList from "../data/stations.meta.json" with { type: "json" }; +// Build-time: emit slim summaries only (see "Build"). Eager, ~0.9 MB. +import summaries from "../data/stations.summary.json" with { type: "json" }; -export const stationsMeta: StationMeta[] = metaList; -export const metaById = new Map(stationsMeta.map((m) => [m.id, m])); +export const stationsMeta: StationSummary[] = summaries; +export const summaryById = new Map(stationsMeta.map((m) => [m.id, m])); ``` -A loader that resolves the subordinate→reference wrinkle (2,239 of 8,290 -stations borrow their reference's constituents): +A Node pack source needs no database dependency, and holds one open handle: ```ts -export function createLoader(source: StationDataSource) { - const cache = new Map>(); +export function packFileSource( + path: string, + index: PackIndex, +): StationDataSource { + let fh: Promise | undefined; + const handle = () => (fh ??= open(path, "r")); + return { + async get(id) { + const range = index[id]; + if (!range) throw new Error(`Station ${id} not found`); + const bytes = Buffer.allocUnsafe(range.length); + await (await handle()).read(bytes, 0, range.length, range.offset); + return JSON.parse(bytes.toString("utf8")); + }, + async close() { + if (fh) await (await fh).close(); + }, + }; +} +``` - function load(id: string): Promise { - let pending = cache.get(id); - if (!pending) { - pending = resolve(id); - cache.set(id, pending); - } - return pending; - } +The loader must not retain parsed records by default. It resolves the +subordinate→reference wrinkle using the subordinate's own record, whose +`offsets.reference` names the reference (so no `ref` field is needed in slim +metadata): - async function resolve(id: string): Promise { - const meta = metaById.get(id); +```ts +export function createLoader(source: StationDataSource) { + async function load(id: string): Promise { + const meta = summaryById.get(id); if (!meta) throw new Error(`Station ${id} not found`); - const data = await source.get(id); - // Subordinate stations predict from their reference's harmonics/datums. - if (meta.type === "subordinate" && meta.ref) { - const ref = await load(meta.ref); - return { - ...meta, - ...data, - datums: ref.datums, - harmonic_constituents: ref.harmonic_constituents, - }; + const data = await source.get(id); // subordinate: own offsets + empty harmonics + // Subordinate stations predict from their reference's harmonics/datums, + // applying their own offsets. One extra read for the reference. + if (meta.type === "subordinate" && data.offsets?.reference) { + const ref = await source.get(data.offsets.reference); + data.harmonic_constituents = ref.harmonic_constituents; + data.datums = ref.datums; } return { ...meta, ...data }; } - return { load }; } ``` -`near`/`search`/`bbox` don't change except their element type becomes -`StationMeta` — they already only touch light fields (they map index ids to -positions/names, never harmonics). +`near`/`search`/`bbox` return `StationSummary` — they already only touch light +fields (they map index ids to positions/names, never harmonics). ## Consumer ripple -Resolving a station by id becomes **async**. That's the one real cost. +Because the async API is a **separate `@neaps/tide-database/async` entry**, the +existing sync exports are untouched; consumers _opt in_ where they want the +low-memory path. Resolving a station by id there is **async** — the one real cost. -- `neaps`: `findStation(id)` and the coordinate-based prediction entry points - become `async` (they must load harmonics before predicting). Prediction math - itself stays sync — `station.getTimelinePrediction(...)` is unchanged once you - hold a loaded `Station`. -- `@neaps/api`: route handlers already run in Express and can `await`. The - handlers become `res.json(await station.getTimelinePrediction(...))`. -- `openapi.ts`: today it does `stations.flatMap(...)` at module top level just to - build the datum enum — that alone forces the full parse. Switch to the fixed - oceanographic datum list (or derive from metadata), so importing the API no - longer touches heavy data. +- `neaps`: add async variants (or an async build) of `findStation` and the + coordinate prediction entry points that `await getStation(...)` before + predicting. The prediction math stays sync once you hold a `Station`. The + existing sync exports remain for consumers still on the eager entry. +- `@neaps/api`: route handlers already `await`, so they call the async resolver + directly — `res.json(await getTimelinePrediction(...))`. +- `openapi.ts`: **done in Phase 0** — it imports the build-time `datums` constant + (exported by this package) instead of scanning every station's datums, which + previously forced the whole database to parse at module load. ## Build -`tsdown`/vite step emits two things instead of one eager glob: - -1. `data/stations.meta.json` — the 1.5 MB metadata array (bundled). -2. The per-station heavy JSON — already exists as `data//.json`; - ship it in the npm package (`fsSource`) and/or publish to R2 / a release - asset (`httpSource`/`r2Source`), stable-named per release like PR #92 does - for the tileset. +The build emits: + +1. **Slim summaries** — `StationSummary[]`, bundled into the JS entry (~0.9 MB). +2. **A byte-range index** — id → `{ offset, length }`, bundled into the JS + (~0.3 MB), produced by the same pass that writes the pack so the two can't + drift. +3. **`stations.pack`** — the heavy JSON records concatenated in deterministic + station-id order, shipped as an npm asset and/or published to R2 / a stable + release asset. + +Offsets are measured in **UTF-8 bytes**, not JS string lengths (station names +carry accents). The build must (a) assert every id is `/` and +that the derived source-id equals the record's `source.id` (the invariant the +`source.id` derivation relies on), and (b) read every indexed record back and +parse it to verify the index. + +## Garbage collection + +Garbage collection depends on reachability, not whether an operation is called +"lazy." + +In Phase 0, module-level exports retain `meta`, `allStations`, `stations`, +`stationsById`, and the complete `heavy: string[]` for the lifetime of the +module. Accessing a getter parses one heavy JSON string. Because the getter does +not cache its result, the parsed harmonics/datums can be collected after the +caller releases them; the original JSON string cannot, because `heavy` still +references it. If a consumer walks every station but does not retain the getter +results, the temporary parsed objects are collectible. + +JavaScript modules themselves are cached and normally cannot be unloaded from a +running process or realm. Dynamically importing one JSON module per station +therefore does not provide reliable eviction: every imported JSON module may +remain in the module cache. + +With the pack source, the read buffer and parsed station can be collected once +the caller releases the returned station, provided the loader does not cache +it. An optional bounded LRU cache can be added later if measurements show that +repeated reads matter. An unbounded `Map` would eventually recreate the current +memory problem. + +After collection, V8 commonly keeps heap pages reserved for future allocations, +so RSS may not fall even though the memory is reusable and no longer counts as +live objects. + +## Alternatives considered + +| Format | Lazy lookup | Assessment | +| ------------------------------------------------------- | ----------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Indexed JSON pack | Yes | Recommended: smallest portable implementation | +| SQLite | Yes | Excellent for Node, but adds native/WASM and HTTP-VFS complexity across runtimes | +| ZIP, one entry per station | Yes | Works, but needs ZIP parsing and its central directory; little benefit over the pack | +| TAR | Only with a separate index | The custom index is still needed, so use the simpler pack layout | +| CBOR/MessagePack records | Yes, with offsets | Potentially smaller, but adds a codec before JSON has proved inadequate | +| One large JSON object | No | Parsing normally materializes the entire dataset | +| Dynamic JSON imports (`import.meta.glob` `eager:false`) | Loads on demand, but does not evict | Tested: 8 MB baseline, but emits 8,290 chunk files, bundles to ~31 MB on Workers (over the limit), and the module cache climbs to ~19 MB after 1,000 distinct loads and never releases | + +PMTiles remains appropriate for geographic/map access, but it is awkward as the +canonical id-keyed prediction store. ## Versioning -This changes the shape and sync-ness of the public API, so it's a **major** -bump. Migration aids: - -- Keep `bundledSource()` so existing offline/sync consumers can opt back into the - old all-in-memory behavior with one line. -- Export `stationsMeta` (sync, light) as the replacement for most `stations` - uses (search results, lists, maps) — those never needed harmonics. +The async API ships as a new `@neaps/tide-database/async` subpath export, so the +main entry and its `stations`/`search`/prediction exports are unchanged — this is +an **additive minor**, not a breaking major. Consumers migrate at their own pace +by importing `/async`. The eager entry keeps its Phase 0 behavior indefinitely +for offline/sync use; there is no forced deprecation. ## Relationship to the vector tileset (PR #92) @@ -193,17 +324,14 @@ and `@neaps/api`. That's too slow for the OOM. A smaller change lands first and fixes the memory crisis without touching the sync contract: Bundle the heavy data as **unparsed JSON strings** instead of live objects, and -parse one station on first access: +parse one station when its heavy fields are accessed: ```ts -// Build emits stations.meta.json (parsed eagerly, ~15-20 MB heap) and a -// data map of id -> raw JSON string (kept as strings, ~23 MB, never parsed -// until touched). Vite: import.meta.glob("./**/*.json", { query: "?raw", ... }). -import dataStrings from "../data/stations.data.js"; // { [id]: string } +// Build-time macros inline metadata objects and one raw JSON string per station. +import dataStrings from "../data/stations.data.js"; function attachLazyData(meta: StationMeta): Station { - let parsed: StationData | undefined; - const load = () => (parsed ??= JSON.parse(dataStrings[meta.id])); + const load = () => JSON.parse(dataStrings[meta.id]); return Object.defineProperties( { ...meta }, { @@ -224,13 +352,16 @@ parses. `near`/`bbox` read only metadata and never trigger a parse. The which geo/id-only consumers (like the plugin) never make. Measured result (import + one `nearest()`, GC'd): heap **118 MB → 69 MB** -(84 MB once text search is used), all 31,859 tests pass, fully synchronous, a -tide-database-only change (+ rebuild + republish). The heavy data still ships -in the bundle as ~17 MB of strings, so this is the floor for a bundled+offline -database; the async pluggable-source version below is what gets it to ~15 MB and -shrinks the edge bundle. RSS stays high on a machine with abundant RAM (V8 keeps -its peak reservation), but the reported crash is a V8 _heap-limit_ OOM, and -under a configured `--max-old-space-size` the process now stays well within it. +(84 MB once text search is used), all 31,862 tests pass, fully synchronous, a +tide-database-only change (+ rebuild + republish). The heavy data still ships in +the bundle as ~17 MB of strings, so this is the floor for a bundled+offline +database. The async pack entry below drops the resident data entirely: its at-rest +footprint is slim metadata (~0.9 MB → ~10 MB heap) + geo index (~66 KB) + byte +index (~0.3 MB), i.e. **~10–15 MB**, with heavy data read from the pack on demand +and text search's 15 MB only if used. RSS stays high on a machine with abundant +RAM (V8 keeps its peak reservation), but the reported crash is a V8 _heap-limit_ +OOM, and under a configured `--max-old-space-size` the process stays well within +it. A companion `@neaps/api` change is required: `openapi.ts` builds its datum enum with `stations.flatMap((s) => Object.keys(s.datums))`, which touches every @@ -239,12 +370,13 @@ station and re-triggers a full parse. tide-database now exports a build-time ## Rollout order -0. **(interim, urgent)** Defer parsing — heavy data bundled as strings, parsed - per station on access. Sync API unchanged. Fixes the OOM. -1. Build emits `stations.meta.json` + keeps per-station files; add - `stationsMeta` export alongside the existing `stations` (no breakage yet). -2. Add `StationDataSource` + `createLoader`; `fsSource` for Node, `httpSource` - for edge. -3. Make `neaps` station resolution async; update `@neaps/api` handlers to await. -4. Point the tides API's Worker at `httpSource`/`r2Source`; drop `bundledSource`. -5. Deprecate the eager `stations` export (major bump). +0. **(done)** Phase 0 — defer parsing; heavy data bundled as strings, parsed per + station on access. Sync API unchanged. Fixes the OOM. +1. Build emits slim summaries, the bundled byte-range index, and `stations.pack`. +2. Add the `@neaps/tide-database/async` entry: `stationsMeta`, `getStation`, + async `near`/`nearest`/`bbox`/`search`, `StationDataSource` + `createLoader`, + `packFileSource` (Node) and range-request sources (browser/edge). +3. `neaps`: add async resolution that `await`s `getStation`; `@neaps/api` + handlers call it. The existing sync entry stays untouched. +4. Point the tides API's Worker at `packHttpSource`/`packObjectSource`; signalk + uses `packFileSource` (offline). From 3a91ee9c3a949cba709cab30daae6c4469f32e77 Mon Sep 17 00:00:00 2001 From: Brandon Keepers Date: Wed, 15 Jul 2026 12:29:22 -0400 Subject: [PATCH 3/4] Store prediction data in a pack file read per record; ship node + browser builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Importing @neaps/tide-database eagerly parsed all 8,290 stations into live JS objects — 118 MB heap / 663 MB RSS. That OOMs memory-constrained consumers (signalk-tides on a Victron Cerbo GX, openwatersio/signalk-tides#103, a V8 heap-limit crash) and dominates the tides API's serverless cold-start CPU. Move the prediction data (harmonic_constituents, datums, epoch) out of the JavaScript heap: - A build step (scripts/generate-pack.mjs) writes stations.pack — the records concatenated as UTF-8 JSON — plus a bundled byte-range index (id -> [offset, length]). Metadata (identity + offsets/source/etc) stays inlined as object literals. - The Node build opens the pack once and readSyncs only the bytes for the station being loaded; nothing is held resident (external stays ~2 MB, and the OS page-caches touched pages). The read loops until the full range is filled so no uninitialized bytes reach JSON.parse. - The package is publicly distributed and may run in a browser (no filesystem), so a browser build bundles the records as JSON strings instead. The source is swapped per build behind the #station-data subpath import; exports conditions select dist/node vs dist/browser. The sync API is unchanged; subordinate stations still resolve to their reference's record (validated at build time). The text index is built lazily on first search() and its serialized string is freed afterward. A build-time `datums` export replaces @neaps/api's all-stations datum scan. Result: import + a nearest() lookup is 35.8 MB heap (prediction data off the heap) vs 118 MB before; reads all 8,290 stations under --max-old-space-size=40. ESM only: kdbush and geokdbush are ESM-only packages that can't be required cleanly from CJS, and all first-party consumers use ESM. A post-build smoke test (scripts/smoke.mjs) imports both built ESM entries, checks reference + subordinate resolution, and asserts the browser bundle has no node:fs — so a broken artifact fails the build. All 31,862 tests pass. Refs openwatersio/signalk-tides#103 --- .gitignore | 3 + docs/lazy-loading.md | 542 +++++++++++++----------------------- package.json | 22 +- scripts/copy-pack.mjs | 14 + scripts/generate-pack.mjs | 76 +++++ scripts/smoke.mjs | 49 ++++ src/search/index.ts | 8 +- src/station-bundle.ts | 43 +-- src/station-data.browser.ts | 16 ++ src/station-data.ts | 31 +++ src/stations.ts | 54 ++-- src/types.ts | 15 +- tsconfig.json | 2 +- tsdown.config.ts | 19 +- 14 files changed, 471 insertions(+), 423 deletions(-) create mode 100644 scripts/copy-pack.mjs create mode 100644 scripts/generate-pack.mjs create mode 100644 scripts/smoke.mjs create mode 100644 src/station-data.browser.ts create mode 100644 src/station-data.ts diff --git a/.gitignore b/.gitignore index b146c1754..79e6e57be 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ node_modules dist package-lock.json tmp + +# generated by scripts/generate-pack.mjs +src/generated/ diff --git a/docs/lazy-loading.md b/docs/lazy-loading.md index ef4564547..98f8d08c3 100644 --- a/docs/lazy-loading.md +++ b/docs/lazy-loading.md @@ -1,382 +1,224 @@ -# Lazy-loading proposal +# Off-heap prediction-data pack -Status: **Phase 0 implemented** (see the interim section below). The proposed -next step is a new, **additive** `@neaps/tide-database/async` entry point that -keeps heavy station data out of the JavaScript bundle and in a single indexed -pack file. The existing `@neaps/tide-database` entry is unchanged — this is not a -breaking change. +Status: **implemented.** Prediction data (harmonic constituents, datums, epoch) +now lives in a `stations.pack` file. The Node build reads only the bytes for the +station being loaded, holding nothing resident; the browser build bundles the +records. The public API is unchanged and still synchronous. The two builds are +selected by `exports` conditions. ## Problem -[`src/stations.ts`](../src/stations.ts) uses `import.meta.glob(..., { eager: true })` -to inline **every** station's JSON into the bundle at build time. So -`dist/index.js` is ~23 MB (32.7 MB raw across 8,290 files), and _any_ import of -the package realizes all of it — even a health check or a single-station lookup. +`src/stations.ts` previously used `import.meta.glob(..., { eager: true })` to +inline **every** station's JSON into the bundle, and any import realized all of +it as live JS objects — even a health check or a single-station lookup. Measured cost — two symptoms, one cause: -- **CPU:** importing `@neaps/tide-database` is ~358 ms; a warm prediction is +- **CPU:** importing `@neaps/tide-database` was ~358 ms; a warm prediction is ~2 ms. On a platform that cold-starts frequently (the tides API burns ~95% of - its CPU budget), that module evaluation _is_ the bill. Caching only helps the - head of the request distribution; the long tail keeps paying the cold start. + its CPU budget), that module evaluation _is_ the bill. - **Memory:** parsing all 8,290 stations (filtered to ~6,177 quality) into live - JS objects costs **118 MB of heap / 663 MB RSS** (the predictor is 4 MB — the - database is all of it). This - OOMs memory-constrained devices: signalk-tides + JS objects cost **118 MB of heap / 663 MB RSS** (the predictor is 4 MB — the + database is all of it). This OOMs memory-constrained devices: signalk-tides ([#103](https://github.com/openwatersio/signalk-tides/issues/103)) crashes a Victron Cerbo GX with a V8 "Reached heap limit" error, because the 118 MB baseline consumes the constrained heap's headroom. The prediction hot loop - itself does **not** leak — measured flat over 600 iterations (~10 h of plugin - runtime). - -The fix is to stop loading 8,290 stations to answer a request about one. - -## Key insight: identity is ~3% of the data - -Split every station into two tiers: - -| Tier | Fields | Size | Loaded | -| ----------------- | ------------------------------------------------------------------------------ | ----------- | ----------------- | -| **Slim metadata** | id, name, lat, lon, region, country, continent, timezone, type | **~0.9 MB** | bundled, eager | -| **Heavy pack** | harmonic_constituents, datums, offsets, epoch, source, license, chart_datum, … | ~32 MB | pack, per station | - -Slim metadata is everything needed to _summarize_ a station in a search result, -nearby list, or map pin. The heavy pack is what you need to _predict_ at a -station or show its full detail — loaded only when the user opens one. (Does the -user need the source license to decide whether to view a station? No — so it -lives in the pack.) - -`source.id` is **not** stored: every station id is `/` (e.g. -`noaa/8722588`), so it is `id.slice(id.indexOf("/") + 1)`. Verified across all -8,290 stations — every id has that shape and the derived source-id equals the -record's `source.id`. So `findStation` by source id and the text index's -`source.id` field both derive it from the id, with no heavy load and no duplicate -field. The build asserts the invariant. See "Search indexes" below for how the -geo and text indexes fit in. - -## Data model - -```ts -// Bundled for all ~8,290 stations (~0.9 MB), parsed once. Everything the -// search / near / bbox / list paths need to render a summary. -export interface StationSummary { - id: string; // "/" - name: string; - latitude: number; - longitude: number; - region?: string; - country: string; - continent: string; - timezone: string; - type: "reference" | "subordinate"; -} - -// Loaded from the pack on demand: the full station record. -export interface StationData { - harmonic_constituents: HarmonicConstituent[]; - datums: Record; - offsets?: { reference: string; height: unknown; time: unknown }; - epoch?: { start: string; end: string }; - source: { id: string; name: string; url: string }; - license: unknown; - disclaimers: string; - chart_datum: string; -} - -// Summary + data — what prediction and detail views need. Today's `Station`. -export type Station = StationSummary & StationData; -``` - -The async entry exposes exactly these two halves: `stationsMeta: StationSummary[]` -(bundled) and `getStation(id): Promise` (loads the pack record). A -consumer that wants the full metadata — license, source, epoch — loads the pack; -a consumer that only lists or maps stations never does. - -## Proposed storage: an indexed pack - -Ship one `stations.pack` — concatenated UTF-8 JSON records — plus a compact -index (station id → byte offset + length) that is **bundled into the JS**, not -shipped as a separate file: - -```text -stations.pack # concatenated UTF-8 JSON records (npm asset / R2) -index: Record # bundled in the JS entry, ~0.3 MB -``` - -Bundling the index is what keeps the client dependency-free: there is no `.idx` -format to parse, only `pack[offset .. offset+length]` → `JSON.parse`. For example -the index records that `noaa/9414290` occupies a particular byte range in -`stations.pack`; loading it reads and parses only that range. This avoids both -thousands of installed files and keeping the complete heavy dataset in the JS -heap. (The index and pack are produced by the same build and versioned together, -so they cannot drift — see "Build".) - -Do not gzip the complete pack as one stream: retrieving a record near the end -would require decompressing everything before it. Initially, leave the pack -uncompressed. If transfer size later proves important, compress records -individually or divide the file into independently compressed blocks. - -The heavy data lives behind a small injectable interface so the same loader can -work in different runtimes: - -```ts -export interface StationDataSource { - get(id: string): Promise; + itself does **not** leak — measured flat over 600 iterations (~10 h of runtime). + +## Key insight: don't put the data on the heap + +The V8 "Reached heap limit" OOM is governed by `heapUsed` vs +`--max-old-space-size`. The previous build kept all 8,290 records as JavaScript +values on that heap. Anything bundled into JavaScript lands there — measured on +the 20 MB pack: + +| Form | resident cost | +| ----------------------------------------- | --------------------------------------- | +| 8,290 JSON string literals (previous) | ~69 MB heap | +| one base64 string literal, decoded | ~58 MB heap (OOMs under a 48 MB cap) | +| **read by byte range from a file** (this) | **~2 MB external**, nothing on the heap | + +String and base64 literals live in the module's constant pool — on the heap. So +the data ships as a **file**, and the Node build reads only the record for the +station being loaded (`openSync` once, `readSync` its byte range), holding +nothing resident: the OS page-caches the touched pages and can evict them under +pressure. (`readFileSync`-ing the whole pack into a `Buffer` would also stay off +the V8 heap — Buffers are external memory — but it pins ~20 MB resident; reading +per record avoids even that, so `external` stays ~2 MB.) + +## What ships + +- **Metadata** — inlined into the JS as object literals via a build macro + (`createStationMeta`): identity plus `offsets`, `source`, `license`, + `chart_datum`, etc. Everything except the prediction data. +- **`stations.pack`** — the prediction data (`harmonic_constituents`, `datums`, + `epoch`) per station, concatenated as UTF-8 JSON records. Shipped as a package + asset (`dist/node/generated/stations.pack`). +- **A byte-range index** — `id -> [offset, length]` into the pack, generated + alongside the pack and bundled into the JS (`src/generated/pack-index.ts`), so + the reader is dependency-free: slice `pack[offset .. offset+length]` → + `JSON.parse`. Offsets are UTF-8 **bytes** (station names carry multibyte chars). + +## Two builds: Node and browser + +The package is publicly distributed and may be used in a browser, which has no +filesystem. So the prediction-data source is swapped per build behind the +`#station-data` subpath import: + +```jsonc +// package.json +"imports": { + "#station-data": { + "browser": "./src/station-data.browser.ts", // bundled JSON strings + "default": "./src/station-data.ts" // per-record pack reads (Node) + } +}, +"exports": { + ".": { + "browser": "./dist/browser/index.js", + "node": "./dist/node/index.js", + "default": "./dist/browser/index.js" + } } ``` -| Source | Runtime | Backing | -| ----------------------- | ----------------------- | --------------------------------------------------------- | -| `packFileSource(path)` | Node | `fs.read()` of the indexed byte range | -| `packHttpSource(url)` | browser / edge | HTTP Range request for the indexed byte range | -| `packObjectSource(obj)` | Cloudflare Workers | R2 range read | -| `bundledSource()` | anywhere, compatibility | current inlined strings; opt-in and retains the full data | +- **Node** (`station-data.ts`): `openSync` the pack once; `getData(id)` `readSync`s + only that record's byte range and parses it. Nothing resident — `external` stays + ~2 MB and the parsed record is transient. +- **Browser** (`station-data.browser.ts`): the records are bundled as JSON strings + (via the `createStationDataById` macro) and parsed one at a time. This costs + more heap (the strings are on the JS heap), but browsers have no equivalent of + Node's tight `--max-old-space-size` limit and no filesystem to read. -For HTTP, the server must support byte ranges and return a stable versioned -pack. Browser bundlers have no universal way to serve an npm package asset, so -browser/edge consumers supply the pack URL (an R2 or release-asset URL). +Both are **ESM only** and expose the same synchronous `getData(id)`, so +`stations.ts` and everything downstream is identical across builds. tsdown builds +both entries; the browser build never contains `node:fs`. (No CJS build — see +below.) -Slim metadata and the index are always bundled, so `search`/`near`/`bbox`/list -stay synchronous and instant on every runtime. Only heavy loads are async. +## The reader and subordinate stations -## Search indexes (geo + text) - -The two indexes are very different sizes and handled differently: - -| Index | Serialized | Built in memory | Used by | Plan | -| ----------------- | ---------- | --------------- | --------------------- | ------------------------------ | -| geo (KDBush) | ~66 KB | ~66 KB | near / nearest / bbox | bundle, eager — negligible | -| text (MiniSearch) | ~1.5 MB | **~15 MB** | `search()` only | build on first search, or skip | - -`near`/`nearest`/`bbox` resolve coordinates → station ids from the ~66 KB geo -index with zero heavy loading; you then `getStation` only the one(s) you want. - -The text index is the single largest optional cost. A search touches every -station, so it can't be range-read from the pack — it's all-or-nothing. -Therefore it is **built lazily on the first `search()`** (Phase 0 already does -this), and geo/id-only consumers like the plugin never pay it. If even the lazy -15 MB is unwanted, a memory-critical build can drop MiniSearch entirely and do a -linear fuzzy scan over the bundled slim-metadata `name` field — 8,290 names is -small enough to scan interactively at zero extra memory. Its `source.id` field is -derived from the id. - -## Code sketch - -Slim metadata, bundled: +`stations.ts` builds `allStations` from the metadata, attaching lazy getters for +the prediction fields: ```ts -// Build-time: emit slim summaries only (see "Build"). Eager, ~0.9 MB. -import summaries from "../data/stations.summary.json" with { type: "json" }; - -export const stationsMeta: StationSummary[] = summaries; -export const summaryById = new Map(stationsMeta.map((m) => [m.id, m])); -``` - -A Node pack source needs no database dependency, and holds one open handle: - -```ts -export function packFileSource( - path: string, - index: PackIndex, -): StationDataSource { - let fh: Promise | undefined; - const handle = () => (fh ??= open(path, "r")); - return { - async get(id) { - const range = index[id]; - if (!range) throw new Error(`Station ${id} not found`); - const bytes = Buffer.allocUnsafe(range.length); - await (await handle()).read(bytes, 0, range.length, range.offset); - return JSON.parse(bytes.toString("utf8")); +function makeStation(m: StationMeta): Station { + // Subordinate stations predict from their reference's data, applying their own + // offsets; resolve to the reference's record. + const dataId = + m.type === "subordinate" && m.offsets ? m.offsets.reference : m.id; + const station = { ...m } as Station; + Object.defineProperties(station, { + harmonic_constituents: { + enumerable: true, + get: () => getData(dataId).harmonic_constituents, }, - async close() { - if (fh) await (await fh).close(); - }, - }; + datums: { enumerable: true, get: () => getData(dataId).datums }, + epoch: { enumerable: true, get: () => getData(m.id).epoch }, + }); + return station; } ``` -The loader must not retain parsed records by default. It resolves the -subordinate→reference wrinkle using the subordinate's own record, whose -`offsets.reference` names the reference (so no `ref` field is needed in slim -metadata): - -```ts -export function createLoader(source: StationDataSource) { - async function load(id: string): Promise { - const meta = summaryById.get(id); - if (!meta) throw new Error(`Station ${id} not found`); - const data = await source.get(id); // subordinate: own offsets + empty harmonics - // Subordinate stations predict from their reference's harmonics/datums, - // applying their own offsets. One extra read for the reference. - if (meta.type === "subordinate" && data.offsets?.reference) { - const ref = await source.get(data.offsets.reference); - data.harmonic_constituents = ref.harmonic_constituents; - data.datums = ref.datums; - } - return { ...meta, ...data }; - } - return { load }; -} -``` +No caching — a persistent cache on these module-level objects would pull the data +back onto the heap on a process that touches every station. Reading a record is a +`readSync` of its byte range + `JSON.parse` (~µs, and the file is OS-page-cached +after warmup). -`near`/`search`/`bbox` return `StationSummary` — they already only touch light -fields (they map index ids to positions/names, never harmonics). - -## Consumer ripple - -Because the async API is a **separate `@neaps/tide-database/async` entry**, the -existing sync exports are untouched; consumers _opt in_ where they want the -low-memory path. Resolving a station by id there is **async** — the one real cost. - -- `neaps`: add async variants (or an async build) of `findStation` and the - coordinate prediction entry points that `await getStation(...)` before - predicting. The prediction math stays sync once you hold a `Station`. The - existing sync exports remain for consumers still on the eager entry. -- `@neaps/api`: route handlers already `await`, so they call the async resolver - directly — `res.json(await getTimelinePrediction(...))`. -- `openapi.ts`: **done in Phase 0** — it imports the build-time `datums` constant - (exported by this package) instead of scanning every station's datums, which - previously forced the whole database to parse at module load. - -## Build - -The build emits: - -1. **Slim summaries** — `StationSummary[]`, bundled into the JS entry (~0.9 MB). -2. **A byte-range index** — id → `{ offset, length }`, bundled into the JS - (~0.3 MB), produced by the same pass that writes the pack so the two can't - drift. -3. **`stations.pack`** — the heavy JSON records concatenated in deterministic - station-id order, shipped as an npm asset and/or published to R2 / a stable - release asset. +## Search indexes (geo + text) -Offsets are measured in **UTF-8 bytes**, not JS string lengths (station names -carry accents). The build must (a) assert every id is `/` and -that the derived source-id equals the record's `source.id` (the invariant the -`source.id` derivation relies on), and (b) read every indexed record back and -parse it to verify the index. +| Index | Serialized | Built in memory | Used by | Plan | +| ----------------- | ---------- | --------------- | --------------------- | ---------------------------- | +| geo (KDBush) | ~66 KB | ~66 KB | near / nearest / bbox | bundled, eager — negligible | +| text (MiniSearch) | ~1.5 MB | ~15 MB | `search()` only | built lazily on first search | + +`near`/`nearest`/`bbox` resolve coordinates to station ids from the geo index +without touching prediction data. The text index is built on the first `search()` +call, so geo/id-only consumers (like the plugin) never pay its ~15 MB. + +## Build pipeline + +`npm run build`: + +1. `generate` (`scripts/generate-pack.mjs`) — reads `data/**/*.json`, writes + `src/generated/stations.pack` and `src/generated/pack-index.ts` (both + git-ignored). A `pretest` hook runs it too, so the tests and both builds always + have a current pack. +2. `tsdown` — builds `dist/node` and `dist/browser` (both ESM), resolving + `#station-data` per build. +3. `copy-pack` — copies the pack to `dist/node/generated/` so the runtime + `new URL("./generated/stations.pack", import.meta.url)` resolves. +4. `tsc --noEmit` — type-checks src and the tests/examples against the built types. +5. `smoke` (`scripts/smoke.mjs`) — imports the built node and browser ESM entries, + checks a reference and a subordinate station resolve their prediction data, and + asserts the browser bundle has no `node:fs`. Runs on every build so a broken + artifact can't ship. + +## Results + +Node build, import + one `nearest()`, GC'd: + +| | `heapUsed` | `external` | +| ------------------------- | ----------- | ---------- | +| eager (before) | 118 MB | — | +| bundled strings (interim) | 69 MB | — | +| **per-record pack (now)** | **35.8 MB** | 1.9 MB | + +The remaining ~36 MB is the sync API's own cost — 8,290 `Station` objects plus +full metadata and the id maps — not the prediction data, which is never resident +(`external` is ~2 MB, and touching all 8,290 stations keeps it flat). It +reads all 8,290 stations under `--max-old-space-size=40`; the old build needs +~69 MB just to start. All 31,862 tests pass. The browser build works with the +records bundled and contains no `node:fs`. + +Trimming metadata to bare identity (deriving `source.id` from the +`/` id, moving `source`/`license`/`offsets` into the pack) +would cut the ~36 MB further, at the cost of extra reads for detail/subordinate +resolution. Left as a future optimization. ## Garbage collection -Garbage collection depends on reachability, not whether an operation is called -"lazy." - -In Phase 0, module-level exports retain `meta`, `allStations`, `stations`, -`stationsById`, and the complete `heavy: string[]` for the lifetime of the -module. Accessing a getter parses one heavy JSON string. Because the getter does -not cache its result, the parsed harmonics/datums can be collected after the -caller releases them; the original JSON string cannot, because `heavy` still -references it. If a consumer walks every station but does not retain the getter -results, the temporary parsed objects are collectible. - -JavaScript modules themselves are cached and normally cannot be unloaded from a -running process or realm. Dynamically importing one JSON module per station -therefore does not provide reliable eviction: every imported JSON module may -remain in the module cache. - -With the pack source, the read buffer and parsed station can be collected once -the caller releases the returned station, provided the loader does not cache -it. An optional bounded LRU cache can be added later if measurements show that -repeated reads matter. An unbounded `Map` would eventually recreate the current -memory problem. - -After collection, V8 commonly keeps heap pages reserved for future allocations, -so RSS may not fall even though the memory is reusable and no longer counts as -live objects. +GC depends on reachability, not on an operation being called "lazy." + +- The metadata objects, `allStations`, `stations`, and `stationsById` are + module-level and live for the process; the pack itself is never held in memory — + only an open file descriptor and transient per-record read buffers. +- A getter parses one record and returns it. Because getters do not cache, the + parsed object is collectible once the caller releases it; walking every station + without retaining results stays flat. +- A bounded LRU cache could be added later if repeated reads ever measure as hot. + An unbounded `Map` would recreate the original problem by pulling data back onto + the heap. +- After collection V8 keeps heap pages reserved, so RSS may not fall even though + the memory is reusable — but the OOM is a heap-limit error, and `heapUsed` is + what dropped. ## Alternatives considered -| Format | Lazy lookup | Assessment | -| ------------------------------------------------------- | ----------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Indexed JSON pack | Yes | Recommended: smallest portable implementation | -| SQLite | Yes | Excellent for Node, but adds native/WASM and HTTP-VFS complexity across runtimes | -| ZIP, one entry per station | Yes | Works, but needs ZIP parsing and its central directory; little benefit over the pack | -| TAR | Only with a separate index | The custom index is still needed, so use the simpler pack layout | -| CBOR/MessagePack records | Yes, with offsets | Potentially smaller, but adds a codec before JSON has proved inadequate | -| One large JSON object | No | Parsing normally materializes the entire dataset | -| Dynamic JSON imports (`import.meta.glob` `eager:false`) | Loads on demand, but does not evict | Tested: 8 MB baseline, but emits 8,290 chunk files, bundles to ~31 MB on Workers (over the limit), and the module cache climbs to ~19 MB after 1,000 distinct loads and never releases | - -PMTiles remains appropriate for geographic/map access, but it is awkward as the -canonical id-keyed prediction store. - -## Versioning - -The async API ships as a new `@neaps/tide-database/async` subpath export, so the -main entry and its `stations`/`search`/prediction exports are unchanged — this is -an **additive minor**, not a breaking major. Consumers migrate at their own pace -by importing `/async`. The eager entry keeps its Phase 0 behavior indefinitely -for offline/sync use; there is no forced deprecation. - -## Relationship to the vector tileset (PR #92) - -Complementary, not competing. The tileset offloads the **map/search/near** -consumer to the client (rendered from a PMTiles file in R2, zero API CPU). This -proposal fixes the **prediction** path (`/:id/timeline`), which is id-keyed and -can't be served from geo-indexed tiles. Do this first — it removes ~95% of the -API's cold-start cost; ship #92 when you want the map to stop hitting the API at -all. - -## Phase 0 (interim): defer parsing, keep the sync API - -The full proposal makes station resolution async, which ripples through `neaps` -and `@neaps/api`. That's too slow for the OOM. A smaller change lands first and -fixes the memory crisis without touching the sync contract: - -Bundle the heavy data as **unparsed JSON strings** instead of live objects, and -parse one station when its heavy fields are accessed: - -```ts -// Build-time macros inline metadata objects and one raw JSON string per station. -import dataStrings from "../data/stations.data.js"; - -function attachLazyData(meta: StationMeta): Station { - const load = () => JSON.parse(dataStrings[meta.id]); - return Object.defineProperties( - { ...meta }, - { - harmonic_constituents: { - get: () => resolveHarmonics(meta, load), - enumerable: true, - }, - datums: { get: () => resolveDatums(meta, load), enumerable: true }, - }, - ); -} -``` - -`useStation` in the predictor destructures `{ datums, harmonic_constituents }`, -so the getters fire for exactly the one station being predicted — nothing else -parses. `near`/`bbox` read only metadata and never trigger a parse. The -~15 MB MiniSearch text index is also deferred until the first `search()` call, -which geo/id-only consumers (like the plugin) never make. - -Measured result (import + one `nearest()`, GC'd): heap **118 MB → 69 MB** -(84 MB once text search is used), all 31,862 tests pass, fully synchronous, a -tide-database-only change (+ rebuild + republish). The heavy data still ships in -the bundle as ~17 MB of strings, so this is the floor for a bundled+offline -database. The async pack entry below drops the resident data entirely: its at-rest -footprint is slim metadata (~0.9 MB → ~10 MB heap) + geo index (~66 KB) + byte -index (~0.3 MB), i.e. **~10–15 MB**, with heavy data read from the pack on demand -and text search's 15 MB only if used. RSS stays high on a machine with abundant -RAM (V8 keeps its peak reservation), but the reported crash is a V8 _heap-limit_ -OOM, and under a configured `--max-old-space-size` the process stays well within -it. - -A companion `@neaps/api` change is required: `openapi.ts` builds its datum enum -with `stations.flatMap((s) => Object.keys(s.datums))`, which touches every -station and re-triggers a full parse. tide-database now exports a build-time -`datums` constant for it to import instead. - -## Rollout order - -0. **(done)** Phase 0 — defer parsing; heavy data bundled as strings, parsed per - station on access. Sync API unchanged. Fixes the OOM. -1. Build emits slim summaries, the bundled byte-range index, and `stations.pack`. -2. Add the `@neaps/tide-database/async` entry: `stationsMeta`, `getStation`, - async `near`/`nearest`/`bbox`/`search`, `StationDataSource` + `createLoader`, - `packFileSource` (Node) and range-request sources (browser/edge). -3. `neaps`: add async resolution that `await`s `getStation`; `@neaps/api` - handlers call it. The existing sync entry stays untouched. -4. Point the tides API's Worker at `packHttpSource`/`packObjectSource`; signalk - uses `packFileSource` (offline). +| Approach | Verdict | +| ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Indexed pack file, read per record (`readSync`) | **Chosen.** Sync API, nothing resident (~2 MB external), one small dependency-free reader. | +| Indexed pack file, whole file in a Buffer | Off-heap too, but pins ~20 MB resident; per-record read avoids it. | +| Bundle data as JSON string literals (interim "Phase 0") | Works but ~69 MB heap — the strings sit on the heap. | +| Bundle data as one base64 literal → Buffer | Rejected: the base64 literal is on-heap (~58 MB, OOMs under a 48 MB cap). | +| `import.meta.glob({ eager: false })` (dynamic imports) | Tested: 8 MB baseline, but emits 8,290 chunk files, bundles to ~31 MB on Workers, and the module cache climbs to ~19 MB after 1,000 distinct loads and never releases. | +| SQLite (better-sqlite3 / D1 / sql.js) | Overkill for read-only id→blob; native/WASM weight across runtimes. | +| One large JSON object | Parsing materializes the entire dataset. | + +## Future: async / edge source + +The current design is synchronous and covers Node (off-heap pack file) and the +browser (bundled records). It does **not** need an async API. If a future consumer +wants low memory in an environment with neither a filesystem nor room to bundle +20 MB — e.g. the tides API on Cloudflare Workers pulling records from R2 — the same +pack can be served by HTTP/R2 range reads behind an **async** `getStation(id)` +(a new `@neaps/tide-database/async` entry). That would be additive; it is not +required by the Node or browser builds and is deferred until a consumer needs it. + +## ESM only (no CJS build) + +`kdbush` and `geokdbush` are ESM-only packages (no `require` export), so a CJS +build can't `require()` them without a double-wrapped-default interop bug +(`KDBush.from is not a function`). All first-party consumers use ESM, so the +package ships **ESM only** — the `require` condition is removed from `exports`. +Modern Node still lets `require()` load the ESM entry (require-of-ESM); older +CJS-only tooling would need to `import()` it. diff --git a/package.json b/package.json index ec2c6341d..6c61b9280 100644 --- a/package.json +++ b/package.json @@ -18,13 +18,20 @@ "license": "MIT", "author": "Brandon Keepers ", "type": "module", - "main": "./dist/index.cjs", - "module": "./dist/index.js", - "types": "./dist/index.d.cts", + "main": "./dist/node/index.js", + "module": "./dist/node/index.js", + "types": "./dist/node/index.d.ts", + "imports": { + "#station-data": { + "browser": "./src/station-data.browser.ts", + "default": "./src/station-data.ts" + } + }, "exports": { ".": { - "import": "./dist/index.js", - "require": "./dist/index.cjs" + "browser": "./dist/browser/index.js", + "node": "./dist/node/index.js", + "default": "./dist/browser/index.js" }, "./package.json": "./package.json" }, @@ -32,8 +39,11 @@ "doc": "docs" }, "scripts": { - "build": "tsc -b && tsc -p tsconfig.node.json && tsdown", + "generate": "node scripts/generate-pack.mjs", + "build": "npm run generate && tsdown && node scripts/copy-pack.mjs && tsc -p tsconfig.json --noEmit && tsc -p tsconfig.node.json && npm run smoke", + "smoke": "node scripts/smoke.mjs", "prepare": "npm run build", + "pretest": "npm run generate", "test": "vitest -r test", "lint": "prettier --check .", "format": "prettier --write ." diff --git a/scripts/copy-pack.mjs b/scripts/copy-pack.mjs new file mode 100644 index 000000000..45dc729e7 --- /dev/null +++ b/scripts/copy-pack.mjs @@ -0,0 +1,14 @@ +// Copies the generated pack next to the bundled Node entry so the runtime +// `new URL("./generated/stations.pack", import.meta.url)` resolves in dist. +// Only the Node build reads the pack; the browser build bundles the data. +import { mkdirSync, copyFileSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +mkdirSync(join(root, "dist", "node", "generated"), { recursive: true }); +copyFileSync( + join(root, "src", "generated", "stations.pack"), + join(root, "dist", "node", "generated", "stations.pack"), +); +console.log("copied stations.pack -> dist/node/generated/"); diff --git a/scripts/generate-pack.mjs b/scripts/generate-pack.mjs new file mode 100644 index 000000000..c152d0129 --- /dev/null +++ b/scripts/generate-pack.mjs @@ -0,0 +1,76 @@ +// Generates the prediction-data pack the Node build reads at runtime. +// +// Emits two files under src/generated/ (git-ignored, regenerated on build/test): +// stations.pack concatenated UTF-8 JSON records {harmonic_constituents, datums, epoch} +// pack-index.ts id -> [byteOffset, byteLength] into the pack +// +// The pack ships as a package asset and is loaded into an off-heap Buffer, so +// importing tide-database no longer holds every station's prediction data on the +// V8 heap. Offsets are UTF-8 byte offsets (station names carry multibyte chars). + +import { readFileSync, writeFileSync, mkdirSync, readdirSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +const dataDir = join(root, "data"); +const outDir = join(root, "src", "generated"); + +function walk(dir) { + return readdirSync(dir, { withFileTypes: true }).flatMap((e) => { + const p = join(dir, e.name); + return e.isDirectory() ? walk(p) : p.endsWith(".json") ? [p] : []; + }); +} + +// Sorted for a deterministic pack layout. +const files = walk(dataDir).sort(); + +const chunks = []; +const index = {}; +const references = []; +let offset = 0; + +for (const file of files) { + const id = file.slice(dataDir.length + 1).replace(/\.json$/, ""); + const s = JSON.parse(readFileSync(file, "utf8")); + if (s.type === "subordinate" && s.offsets?.reference) { + references.push([id, s.offsets.reference]); + } + const record = JSON.stringify({ + harmonic_constituents: s.harmonic_constituents ?? [], + datums: s.datums ?? {}, + epoch: s.epoch, + }); + const buf = Buffer.from(record, "utf8"); + index[id] = [offset, buf.length]; + chunks.push(buf); + offset += buf.length; +} + +// Fail the build if any subordinate points at a missing reference — otherwise it +// would only surface at prediction time as a runtime error. +for (const [id, reference] of references) { + if (!(reference in index)) { + throw new Error( + `Station ${id} references missing reference station ${reference}`, + ); + } +} + +const pack = Buffer.concat(chunks); + +mkdirSync(outDir, { recursive: true }); +writeFileSync(join(outDir, "stations.pack"), pack); +writeFileSync( + join(outDir, "pack-index.ts"), + "// Generated by scripts/generate-pack.mjs — do not edit.\n" + + "export const packIndex: Record = " + + JSON.stringify(index) + + ";\n", +); + +console.log( + `generated pack: ${(pack.length / 1048576).toFixed(1)} MB, ${files.length} stations, ` + + `index ${(Buffer.byteLength(JSON.stringify(index)) / 1048576).toFixed(2)} MB`, +); diff --git a/scripts/smoke.mjs b/scripts/smoke.mjs new file mode 100644 index 000000000..6ca4e2d97 --- /dev/null +++ b/scripts/smoke.mjs @@ -0,0 +1,49 @@ +// Smoke-tests the built artifacts (not src), so a broken published package can't +// slip through. Runs after build. Imports the Node ESM entry and the browser +// ESM entry, checks a reference and a subordinate station resolve their +// prediction data, and asserts the browser bundle contains no node:fs. +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +const dist = new URL("../dist/", import.meta.url); + +function check(db, label) { + const ref = db.stations.find( + (s) => s.type === "reference" && s.harmonic_constituents.length > 0, + ); + assert.ok(ref, `${label}: a reference station with harmonics`); + assert.ok( + ref.harmonic_constituents.length > 0, + `${label}: reference harmonics`, + ); + assert.ok(Object.keys(ref.datums).length > 0, `${label}: reference datums`); + + const sub = db.stations.find((s) => s.type === "subordinate" && s.offsets); + assert.ok(sub, `${label}: a subordinate station`); + assert.ok( + sub.harmonic_constituents.length > 0, + `${label}: subordinate resolves reference harmonics`, + ); + + assert.ok(db.search("seattle")[0]?.name, `${label}: search`); + assert.ok( + db.nearest({ latitude: 47.6, longitude: -122.3 }), + `${label}: nearest`, + ); + assert.ok(db.datums.length > 0, `${label}: datums export`); +} + +check(await import(new URL("node/index.js", dist)), "node ESM"); +check(await import(new URL("browser/index.js", dist)), "browser ESM"); + +const browserSrc = readFileSync( + fileURLToPath(new URL("browser/index.js", dist)), + "utf8", +); +assert.ok( + !/["']node:fs["']|require\(["']fs["']\)/.test(browserSrc), + "browser bundle must not reference node:fs", +); + +console.log("smoke: node ESM + browser ESM OK"); diff --git a/src/search/index.ts b/src/search/index.ts index 33398a2fc..0a2eda991 100644 --- a/src/search/index.ts +++ b/src/search/index.ts @@ -49,10 +49,14 @@ const geoIndex = loadGeoIndex(await createGeoIndex()); // search(), so defer building it until the first text search. The serialized // index string (~1.5 MB) is still inlined at build time; only the expensive // loadTextIndex build is deferred. -const textIndexData = await createTextIndex(); +let textIndexData: string | undefined = await createTextIndex(); let textIndex: ReturnType | undefined; function getTextIndex() { - return (textIndex ??= loadTextIndex(textIndexData)); + if (!textIndex) { + textIndex = loadTextIndex(textIndexData!); + textIndexData = undefined; // free the ~1.5 MB serialized string for GC + } + return textIndex; } function createFilter( diff --git a/src/station-bundle.ts b/src/station-bundle.ts index a6f553a4c..46b49d38e 100644 --- a/src/station-bundle.ts +++ b/src/station-bundle.ts @@ -30,12 +30,14 @@ function readAll(): { id: string; data: StationData }[] { import: "default", base: "../data", }); - // Object.entries preserves the glob's sorted key order, so the metadata array, - // the heavy array, and the geo/text indexes all share one station ordering. - return Object.entries(modules).map(([path, data]) => ({ - id: path.replace(/^\.\//, "").replace(/\.json$/, ""), - data, - })); + // Sort explicitly so the metadata array and the geo/text indexes share one + // deterministic station order regardless of the bundler's glob implementation. + return Object.entries(modules) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([path, data]) => ({ + id: path.replace(/^\.\//, "").replace(/\.json$/, ""), + data, + })); } /** @@ -49,7 +51,8 @@ export function createDatumEnum(): string[] { if (data.datums) for (const key of Object.keys(data.datums)) datums.add(key); } - return [...datums]; + // Sorted for deterministic builds (downstream may embed this, e.g. OpenAPI enums). + return [...datums].sort(); } /** Light metadata for every station, in bundle order. Used at build time. */ @@ -73,17 +76,19 @@ export function createStationMeta(): StationMeta[] { } /** - * Heavy fields (harmonic_constituents, datums, epoch) as an array of per-station - * JSON strings, inlined as an array of string literals. The strings are the live - * data; each is JSON.parsed on demand, so importing the bundle never - * materializes all stations' harmonics at once. + * Prediction data (harmonic_constituents, datums, epoch) keyed by station id, + * each a JSON string. Used by the browser build, which inlines this as an object + * of string literals and parses one record on demand. (The Node build reads the + * same records from an off-heap pack file instead — see station-data.ts.) */ -export function createStationHeavy(): string[] { - return readAll().map(({ data }) => - JSON.stringify({ - harmonic_constituents: data.harmonic_constituents ?? [], - datums: data.datums ?? {}, - epoch: data.epoch, - }), - ); +export function createStationDataById(): Record { + const data: Record = {}; + for (const { id, data: station } of readAll()) { + data[id] = JSON.stringify({ + harmonic_constituents: station.harmonic_constituents ?? [], + datums: station.datums ?? {}, + epoch: station.epoch, + }); + } + return data; } diff --git a/src/station-data.browser.ts b/src/station-data.browser.ts new file mode 100644 index 000000000..b1be7a903 --- /dev/null +++ b/src/station-data.browser.ts @@ -0,0 +1,16 @@ +// Browser prediction-data source. Selected via the "#station-data" subpath +// import for the browser build (which has no filesystem). The records are +// bundled as JSON strings and parsed one at a time on demand. This costs more +// heap than the Node pack (the strings are on the JS heap), but browsers have no +// equivalent of Node's tight --max-old-space-size limit, and keeping the API +// synchronous matters more than the extra megabytes here. +import { createStationDataById } from "./station-bundle.js" with { type: "macro" }; +import type { PredictionData } from "./types.js"; + +const data: Record = createStationDataById(); + +export function getData(id: string): PredictionData { + const record = data[id]; + if (!record) throw new Error(`No data record for station ${id}`); + return JSON.parse(record); +} diff --git a/src/station-data.ts b/src/station-data.ts new file mode 100644 index 000000000..de3479c75 --- /dev/null +++ b/src/station-data.ts @@ -0,0 +1,31 @@ +// Node prediction-data source. Selected via the "#station-data" subpath import +// for the Node build (and by tests). One file handle is held open; each lookup +// reads only that station's byte range from the pack — the file is never loaded +// whole. So the process holds none of the ~6,000 stations' prediction data (the +// OS page-caches the touched pages, which it can evict under memory pressure), +// and the parsed record is transient. +import { openSync, readSync } from "node:fs"; +import { packIndex } from "./generated/pack-index.js"; +import type { PredictionData } from "./types.js"; + +const fd = openSync(new URL("./generated/stations.pack", import.meta.url), "r"); + +export function getData(id: string): PredictionData { + const range = packIndex[id]; + if (!range) throw new Error(`No data record for station ${id}`); + const [offset, length] = range; + const buffer = Buffer.allocUnsafe(length); + // readSync may return a short read; loop until the whole range is filled so no + // uninitialized bytes from allocUnsafe reach JSON.parse. + let read = 0; + while (read < length) { + const n = readSync(fd, buffer, read, length - read, offset + read); + if (n === 0) { + throw new Error( + `Short read for station ${id}: ${read} of ${length} bytes`, + ); + } + read += n; + } + return JSON.parse(buffer.toString("utf8")); +} diff --git a/src/stations.ts b/src/stations.ts index f2700528c..5cf6bd4ff 100644 --- a/src/stations.ts +++ b/src/stations.ts @@ -1,65 +1,45 @@ -import type { - HarmonicConstituent, - Station, - StationData, - StationMeta, -} from "./types.js"; +import type { Station, StationMeta } from "./types.js"; import { createStationMeta } from "./station-bundle.js" with { type: "macro" }; -import { createStationHeavy } from "./station-bundle.js" with { type: "macro" }; import { createDatumEnum } from "./station-bundle.js" with { type: "macro" }; +import { getData } from "#station-data"; import quality from "../quality.json" with { type: "json" }; /** All datum keys present across the database (e.g. "MLLW", "MSL", "NAVD88"). */ export const datums: string[] = createDatumEnum(); -// Metadata is inlined as object literals (~15 MB of live objects). Heavy fields -// are inlined as an array of per-station JSON string literals and parsed on -// demand, so importing this module no longer materializes all 6,000+ stations' -// harmonics (which cost ~118 MB of heap / ~660 MB RSS eagerly). +// Metadata (identity + offsets/source/etc) is inlined as object literals. The +// prediction data (harmonic_constituents, datums, epoch) comes from a per-runtime +// source (#station-data): an off-heap pack file on Node, bundled strings in the +// browser. Either way, importing this module holds no station data on the heap — +// a station's record is parsed only when its prediction fields are accessed. const meta: StationMeta[] = createStationMeta(); -const heavy: string[] = createStationHeavy(); -const indexById = new Map(meta.map((m, i) => [m.id, i] as const)); - -interface HeavyFields { - harmonic_constituents: HarmonicConstituent[]; - datums: Record; - epoch?: StationData["epoch"]; -} - -function parseHeavy(index: number): HeavyFields { - return JSON.parse(heavy[index]!); -} - -function makeStation(m: StationMeta, index: number): Station { +function makeStation(m: StationMeta): Station { // Subordinate stations predict from their reference station's harmonics and - // datums (their own offsets still apply). Resolve to the reference's heavy - // data; fall back to self if the reference is somehow missing. - const dataIndex = - m.type === "subordinate" && m.offsets - ? (indexById.get(m.offsets.reference) ?? index) - : index; + // datums (their own offsets still apply); resolve to the reference's record. + const dataId = + m.type === "subordinate" && m.offsets ? m.offsets.reference : m.id; const station = { ...m } as Station; - // Getters keep the sync API: reading these fields parses one station's heavy - // blob. No caching — a persistent cache on these module-level objects would - // grow back toward the full 118 MB on a process that touches every station. + // Getters keep the sync API: reading these fields parses one station's record. + // No caching — a persistent cache on these module-level objects would pull the + // heavy data back onto the heap. Object.defineProperties(station, { harmonic_constituents: { enumerable: true, configurable: true, - get: () => parseHeavy(dataIndex).harmonic_constituents, + get: () => getData(dataId).harmonic_constituents, }, datums: { enumerable: true, configurable: true, - get: () => parseHeavy(dataIndex).datums, + get: () => getData(dataId).datums, }, epoch: { enumerable: true, configurable: true, - get: () => parseHeavy(index).epoch, + get: () => getData(m.id).epoch, }, }); diff --git a/src/types.ts b/src/types.ts index b7dbbf8ee..5a2a42c7a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -70,9 +70,10 @@ export interface Station extends StationData { id: string; } -// The light fields, bundled eagerly for all stations (~1.5 MB). Everything the -// search/geo/list paths need. The heavy fields (harmonic_constituents, datums, -// epoch) are loaded lazily per station — see station-bundle.ts. +// The light fields, bundled eagerly for all stations (~1.5 MB serialized; +// ~15 MB as live objects on the heap). Everything the search/geo/list paths +// need. The prediction data (harmonic_constituents, datums, epoch) is loaded +// per station from the pack — see station-data.ts. export type StationMetaKey = | "name" | "latitude" @@ -90,3 +91,11 @@ export type StationMetaKey = | "offsets"; export type StationMeta = { id: string } & Pick; + +// The lazily-loaded prediction data, resolved per station from the data source +// (an off-heap pack file on Node, bundled JSON strings in the browser). +export interface PredictionData { + harmonic_constituents: HarmonicConstituent[]; + datums: Record; + epoch?: StationData["epoch"]; +} diff --git a/tsconfig.json b/tsconfig.json index d3e5c1f73..7207f4136 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ "compilerOptions": { "rootDir": "./src", "outDir": "./dist", - "types": ["vite/client"], + "types": ["vite/client", "node"], "declarationMap": true, "tsBuildInfoFile": "./dist/.tsbuildinfo" }, diff --git a/tsdown.config.ts b/tsdown.config.ts index 0392b66d6..8307a2953 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -1,13 +1,22 @@ import { defineConfig } from "tsdown"; import macros from "unplugin-macros/rolldown"; -export default defineConfig({ - exports: true, +// Two builds. The Node build resolves `#station-data` to station-data.ts (reads +// the off-heap pack file); the browser build resolves it to +// station-data.browser.ts (bundled JSON strings). package.json `exports` +// conditions select between dist/node and dist/browser. +const base = { + entry: ["./src/index.ts"], dts: true, minify: true, - format: ["cjs", "esm"], sourcemap: true, target: "es2020", - platform: "neutral", plugins: [macros()], -}); +}; + +// ESM only. kdbush and geokdbush are ESM-only, so a CJS build can't require them +// cleanly; all first-party consumers use ESM. `#station-data` resolves per build. +export default defineConfig([ + { ...base, platform: "neutral", format: ["esm"], outDir: "dist/node" }, + { ...base, platform: "browser", format: ["esm"], outDir: "dist/browser" }, +]); From c3207a0fa8fcb06be4e2e6d85848945aa1e500de Mon Sep 17 00:00:00 2001 From: Brandon Keepers Date: Tue, 28 Jul 2026 12:32:39 -0400 Subject: [PATCH 4/4] Bump version to 0.9 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6c61b9280..0a33b8eca 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@neaps/tide-database", - "version": "0.8.0", + "version": "0.9.0", "description": "A public database of tide harmonics", "keywords": [ "tides",