From e7d57cdf428215161fb1f8baa748cd31ce7e55bd Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 01:22:17 +0000 Subject: [PATCH 01/16] docs(intelligence): extension workspace scaffold + evidence-claim RFC + architecture docs New self-contained npm-workspaces root at intelligence/ (10 private packages, never published) implementing the PACT.TAILOR intelligence network as a Tailor extension per GOVERNANCE.md D2=B. No files under spec/ are touched; the design record is docs/v2-prep/rfc-intelligence-fact-evidence-extension.md, which formalises the built-in fact type's informal {claim, evidence, tier, sources} payload as an evidence-carrying claim envelope (registration deferred to a follow-up PR under the resource-type registry's own process). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014yVGqBtvM1zNAd9tvUVNzm --- .gitignore | 3 + ...fc-intelligence-fact-evidence-extension.md | 154 + intelligence/README.md | 94 + intelligence/docs/ARCHITECTURE.md | 90 + intelligence/docs/DATA-MODEL.md | 116 + intelligence/docs/GLOBE.md | 53 + intelligence/docs/IDENTIFIERS.md | 83 + intelligence/docs/LEGAL-LICENSING.md | 53 + intelligence/docs/SDUI.md | 63 + intelligence/docs/STORAGE.md | 49 + intelligence/package-lock.json | 5300 +++++++++++++++++ intelligence/package.json | 25 + intelligence/packages/api/package.json | 26 + intelligence/packages/api/src/index.ts | 2 + intelligence/packages/api/tsconfig.json | 8 + intelligence/packages/connectors/package.json | 28 + intelligence/packages/connectors/src/index.ts | 2 + .../packages/connectors/tsconfig.json | 8 + intelligence/packages/globe/package.json | 22 + intelligence/packages/globe/src/main.ts | 2 + intelligence/packages/globe/tsconfig.json | 12 + intelligence/packages/graph/package.json | 21 + intelligence/packages/graph/src/index.ts | 2 + intelligence/packages/graph/tsconfig.json | 8 + intelligence/packages/intel/package.json | 23 + intelligence/packages/intel/src/index.ts | 2 + intelligence/packages/intel/tsconfig.json | 8 + intelligence/packages/ontology/package.json | 22 + intelligence/packages/ontology/src/index.ts | 3 + intelligence/packages/ontology/tsconfig.json | 8 + .../packages/pact-bridge/package.json | 22 + .../packages/pact-bridge/src/index.ts | 2 + .../packages/pact-bridge/tsconfig.json | 8 + intelligence/packages/registry/package.json | 22 + intelligence/packages/registry/src/index.ts | 2 + intelligence/packages/registry/tsconfig.json | 8 + intelligence/packages/state/package.json | 22 + intelligence/packages/state/src/index.ts | 2 + intelligence/packages/state/tsconfig.json | 8 + intelligence/packages/store/package.json | 21 + intelligence/packages/store/src/index.ts | 2 + intelligence/packages/store/tsconfig.json | 8 + intelligence/tsconfig.base.json | 17 + 43 files changed, 6434 insertions(+) create mode 100644 docs/v2-prep/rfc-intelligence-fact-evidence-extension.md create mode 100644 intelligence/README.md create mode 100644 intelligence/docs/ARCHITECTURE.md create mode 100644 intelligence/docs/DATA-MODEL.md create mode 100644 intelligence/docs/GLOBE.md create mode 100644 intelligence/docs/IDENTIFIERS.md create mode 100644 intelligence/docs/LEGAL-LICENSING.md create mode 100644 intelligence/docs/SDUI.md create mode 100644 intelligence/docs/STORAGE.md create mode 100644 intelligence/package-lock.json create mode 100644 intelligence/package.json create mode 100644 intelligence/packages/api/package.json create mode 100644 intelligence/packages/api/src/index.ts create mode 100644 intelligence/packages/api/tsconfig.json create mode 100644 intelligence/packages/connectors/package.json create mode 100644 intelligence/packages/connectors/src/index.ts create mode 100644 intelligence/packages/connectors/tsconfig.json create mode 100644 intelligence/packages/globe/package.json create mode 100644 intelligence/packages/globe/src/main.ts create mode 100644 intelligence/packages/globe/tsconfig.json create mode 100644 intelligence/packages/graph/package.json create mode 100644 intelligence/packages/graph/src/index.ts create mode 100644 intelligence/packages/graph/tsconfig.json create mode 100644 intelligence/packages/intel/package.json create mode 100644 intelligence/packages/intel/src/index.ts create mode 100644 intelligence/packages/intel/tsconfig.json create mode 100644 intelligence/packages/ontology/package.json create mode 100644 intelligence/packages/ontology/src/index.ts create mode 100644 intelligence/packages/ontology/tsconfig.json create mode 100644 intelligence/packages/pact-bridge/package.json create mode 100644 intelligence/packages/pact-bridge/src/index.ts create mode 100644 intelligence/packages/pact-bridge/tsconfig.json create mode 100644 intelligence/packages/registry/package.json create mode 100644 intelligence/packages/registry/src/index.ts create mode 100644 intelligence/packages/registry/tsconfig.json create mode 100644 intelligence/packages/state/package.json create mode 100644 intelligence/packages/state/src/index.ts create mode 100644 intelligence/packages/state/tsconfig.json create mode 100644 intelligence/packages/store/package.json create mode 100644 intelligence/packages/store/src/index.ts create mode 100644 intelligence/packages/store/tsconfig.json create mode 100644 intelligence/tsconfig.base.json diff --git a/.gitignore b/.gitignore index aa60250..f6e7013 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,6 @@ dist/ # Claude Code session state (per-session worktrees, not part of repo) .claude/ + +# Intelligence workspace runtime output (bronze/silver/gold data lake) +intelligence/var/ diff --git a/docs/v2-prep/rfc-intelligence-fact-evidence-extension.md b/docs/v2-prep/rfc-intelligence-fact-evidence-extension.md new file mode 100644 index 0000000..c3b20f8 --- /dev/null +++ b/docs/v2-prep/rfc-intelligence-fact-evidence-extension.md @@ -0,0 +1,154 @@ +# RFC: `au.tailor.intel.claim` — an evidence-carrying claim payload for the PACT `fact` resource type + +> **Status:** RFC — for discussion. Implementation ships in the `intelligence/` +> extension workspace; **no spec text or registry entry changes in this PR.** +> **Proposed resource type:** `au.tailor.intel.claim` (registration deferred — see §6) +> **Target PACT version:** v2.2 (§14 resource types; §24 Matters unaffected) +> **Owner:** Tailor (extension workspace); registry decision remains with the maintainer +> **Related:** `spec/v2.2/resource-types.yaml` built-in `fact` type; GOVERNANCE.md +> decision D2=B (domain-specific Tailor needs are extensions, not core); issue #34 +> (KG product extraction — explicitly *not* what this workspace is, see §7) +> **Reference implementation:** `intelligence/schemas/inference.schema.json` + +> `intelligence/packages/pact-bridge/` (competing-hypotheses demo against +> `reference-server/`) + +--- + +## 1. Summary + +PACT's built-in `fact` resource type verifies knowledge claims into a knowledge graph, +but its proposal payload — `{ claim, evidence, tier, sources }` — is described in the +registry as a string, with no JSON Schema behind it. `tier` and `sources` are undefined +in spec text; `evidence` has no structure; nothing binds a claim to the observations +that justify it. + +The intelligence workspace needs exactly that binding: its rule is that **derived +intelligence must always point back to its evidence and can never masquerade as direct +telemetry**. This RFC documents the formalisation the workspace implements — a +schema'd, evidence-carrying claim payload — and proposes it, eventually, as a +registered custom resource type `au.tailor.intel.claim`. + +## 2. Motivation + +A live intelligence system produces statements like: + +> Vessel X departing Gladstone is likely carrying metallurgical coal. Confidence 0.91. + +That statement is useful precisely to the degree that its support is inspectable. The +informal `fact` payload can carry the sentence; it cannot carry, in any checkable way, +*which* AIS observations, berth assignments, and structural supply edges support it, +which evidence cuts against it, what rule produced it, or what its status became after +other agents weighed in. + +PACT already solves the adjacent problem for *authority* (hash-chained event log, +`authorization_proof`, mandates). This RFC applies the same discipline to *epistemics*: +claims carry their evidence, their counter-evidence, their method, and their lineage. + +## 3. The payload + +Normative schema: `intelligence/schemas/inference.schema.json`. Shape (informative): + +```json +{ + "inference_id": "infer:2026-08-12:rgt-cargo-01", + "claim": "Vessel MV Example, departing RG Tanna berth 1, is loading metallurgical coal.", + "claim_structured": { + "subject": "vessel:imo:9700000", + "predicate": "carries", + "object": "commodity:coal-metallurgical", + "qualifiers": { "loading_at": "terminal:au-qld:rg-tanna" } + }, + "tier": "B", + "confidence": 0.91, + "status": "INFERRED", + "method": "rule:cargo-inference@1.0.0", + "evidence": [ + { "kind": "observation", "ref": "01J...ULID", "role": "supports" }, + { "kind": "structural", "ref": "rel:terminal-supplies-met-coal", "role": "supports" } + ], + "contrary_evidence": [ + { "kind": "observation", "ref": "01J...ULID2", "role": "contradicts" } + ], + "sources": ["source:ais:aisstream", "source:aemo:nemweb-dispatchis"], + "produced_at": "2026-08-12T03:15:00Z", + "event_time_range": { "from": "2026-08-12T01:00:00Z", "to": "2026-08-12T03:10:00Z" }, + "lineage": { "correlation_id": "corr-...", "in_response_to": null, "prev_hash": "..." }, + "pact": { "fabric_id": "...", "proposal_id": "..." } +} +``` + +Mapping to the built-in `fact` payload is total: `claim` → `claim`, +`evidence ∪ contrary_evidence` → `evidence`, `tier` → `tier`, `sources` → `sources` — +so an `au.tailor.intel.claim` proposal degrades losslessly to a legacy `fact` proposal +for servers that only know the built-in type. The bridge in +`intelligence/packages/pact-bridge/src/fact-mapper.ts` implements this mapping today. + +## 4. Semantics + +- **Evidence tiers:** `A` — the claimed quantity was directly observed; `B` — one-hop + proxy or derivation from direct observations; `C` — multi-hop inference or pattern + match. Tier is about *distance from observation*, confidence is about *strength*; + the two are orthogonal and both required. +- **Status lifecycle:** `INFERRED → CORROBORATED | CONTESTED → VERIFIED | RETRACTED`. + Transitions happen through PACT consensus (propose/object/salience on the claim + field) or through new evidence; records are append-only — a status change is a new + revision, never an edit that loses the contested history. +- **Contrary evidence is first-class.** An objection that cites evidence lands in + `contrary_evidence` and survives resolution. A claim that won consensus still shows + what cut against it. +- **Non-masquerade rule (normative for the workspace):** an inference envelope can + never be written to the observation store, and no observation field is + author-choosable to claim otherwise. Consumers can always distinguish reported from + inferred — this is the payload-level expression of PACT's evidence-backed-claim + intent in §14.5. +- **Lineage** reuses the event-log vocabulary (`correlation_id`, `in_response_to`, + `prev_hash` per §6.1/§6.4) rather than inventing a parallel provenance chain. An + inference chain is auditable with the same tooling as a PACT event chain. + +## 5. Consensus flow (worked example, implemented) + +The Gladstone demo (`intelligence/packages/pact-bridge/`): two agents propose competing +cargo claims for the same vessel — metallurgical coal (0.91) citing berth assignment + +draught-change observations + terminal supply edges, versus thermal coal (0.34) citing +a contrary shipping-schedule observation. One objects to the other's claim; salience +and consensus resolve; the winning inference becomes `CORROBORATED`, the losing one +`CONTESTED`; both retain full evidence and contrary-evidence arrays. Runs offline +against `reference-server/` in `node --test`. + +## 6. Registration (deferred) + +Registering `au.tailor.intel.claim` in `spec/vX.Y/resource-types.yaml` follows the +registry's own process — a dedicated PR against that file, linking the reference +implementation, reviewed by the maintainer. **This workspace's PR deliberately does not +touch the registry** (or any file under `spec/`): per AGENTS.md, agents do not freehand +spec-adjacent surfaces; the registration PR is a follow-up under maintainer sign-off. +Proposed entry (informative draft): + +```yaml +- type: au.tailor.intel.claim + status: registered + field_schema: "claim:{id} — evidence-carrying knowledge-claim identifier" + proposal_payload: "inference envelope per intelligence/schemas/inference.schema.json" + apply_semantics: Claim verified into the knowledge graph with evidence and lineage retained. + terminal_states: [Verified, Retracted] + content_format: application/json + maintainer: TailorAU +``` + +## 7. Relationship to issue #34 + +Issue #34 (extraction of the production KG product into this repo) is owner-only and +security-sensitive. This workspace is **not** that extraction: it is a new extension +implementation under GOVERNANCE.md D2=B, with its own data model and no dependency on +`tailor-app` internals. The only overlap is conceptual — both verify claims into a +knowledge graph — and this RFC is written so the production product could *adopt* the +schema'd payload later if the owner chooses. + +## 8. Compatibility + +- Servers that know only the built-in `fact` type receive the degraded + `{ claim, evidence, tier, sources }` mapping (§3) — no breakage. +- The payload adds no PACT operations, no endpoints, no `_meta` extensions; it rides + the existing propose/object/consensus machinery unchanged. +- Nothing in this RFC is normative for PACT. It becomes normative for the registry + only if/when the §6 registration PR is accepted. diff --git a/intelligence/README.md b/intelligence/README.md new file mode 100644 index 0000000..a48d1d0 --- /dev/null +++ b/intelligence/README.md @@ -0,0 +1,94 @@ +# PACT.TAILOR Intelligence Network + +A real-time model of the physical economy — energy, industry, and trade — built as a +**Tailor extension workspace** on top of the PACT protocol. The machine interface is a +REST + SSE API over a bitemporal knowledge graph; the human interface is a GPU-rendered +globe; the coordination, evidence, and attestation layer is PACT. + +This directory is an *extension implementation* in the sense of `GOVERNANCE.md` +(decision D2=B: domain-specific Tailor needs land as extensions, not PACT core). Nothing +under `spec/` is modified by this workspace. The design record lives at +`docs/v2-prep/rfc-intelligence-fact-evidence-extension.md`. + +## The one rule + +**Intelligence means live observable state — not AI prose.** Every piece of information +in this system belongs to exactly one of three classes, and the class is structurally +unforgeable (each class has its own envelope schema): + +| Class | Envelope | Meaning | +|---|---|---| +| A. Structural | `schemas/entity.schema.json`, `relationship.schema.json` | The world model: what exists and how it connects | +| B. Live telemetry | `schemas/observation.schema.json` | What is being observed right now | +| C. Derived intelligence | `schemas/inference.schema.json` | Interpretations — **must** reference their evidence | + +Derived intelligence can never be written into the observation store, and inferred +cargo/state/utilisation is never presented as reported fact. Where live observability +does not exist, the system records an explicit **observability gap** +(`schemas/observability-gap.schema.json`) — a first-class product output identifying +where Tailor/Sovrgn could deploy owned instrumentation. + +## Honest data status + +This workspace was developed in an environment whose network egress policy blocks all +energy-data endpoints. Consequently: + +- Every connector is built against the *documented* wire format of its source and ships + with fixtures marked `fixture_provenance: synthetic-from-spec`. +- Every source registry record carries `verification_status: documented_only` until the + live-verification harness (`npm run verify:live`) has actually reached the endpoint. +- No claim of live verification is made anywhere in this tree. The replay path used in + tests is the *same code path* as live ingestion (`HttpGate` swaps `ReplayGate` for + `LiveGate`), so passing fixtures is meaningful but is not live proof. + +## Layout + +``` +schemas/ Canonical JSON Schemas (2020-12) for all envelopes +data/ YAML registries: grids, sources, observability gaps + NEM structural seed +packages/ + ontology TS types, ajv validators, ID grammar + registry YAML loaders, cross-reference validation, entity-resolution candidates + store Bronze/silver/gold medallion storage (embedded, zero external services) + graph In-process bitemporal property graph + traversal + state Real-time state engine: observations → current-state projections + connectors Connector SDK + AEMO / AIS / BOM / ENTSO-E / EIA connectors + fixtures + intel Rule-based derived intelligence (utilisation, cargo inference, anomalies) + api Zero-dep REST + SSE server + SDUI composer + pact-bridge Inference → PACT `fact` proposals; competing-hypotheses demo + globe deck.gl GPU globe prototype (Vite) +docs/ Architecture, data model, identifiers, storage, SDUI, globe, licensing +scripts/ End-to-end smoke test +var/ Runtime output (gitignored; bronze/silver/gold live here) +``` + +## Quick start + +```bash +cd intelligence +npm ci +npm run build # builds all packages in dependency order +npm test # per-package node --test suites +npm run validate:data # ajv-validates every schema + every YAML registry record +npm run smoke # fixtures → ingest → state → graph → API → SSE, end to end +``` + +Run the API against replayed fixtures: + +```bash +node packages/connectors/dist/cli.js ingest --replay +node packages/api/dist/cli.js --port 4200 +# then: curl localhost:4200/api/intel/grids/grid:au-nem/summary +``` + +Run the globe dev server (expects the API on :4200): + +```bash +npm run dev -w @pact-tailor/globe +``` + +## Status + +All packages are `private: true` and are **never published to npm** (see repo issue #5). +First depth target: Australia's NEM, with the Gladstone industrial cluster as the worked +exemplar. Global breadth lives in the registries. diff --git a/intelligence/docs/ARCHITECTURE.md b/intelligence/docs/ARCHITECTURE.md new file mode 100644 index 0000000..949853e --- /dev/null +++ b/intelligence/docs/ARCHITECTURE.md @@ -0,0 +1,90 @@ +# Architecture + +The system is a pipeline from raw external observation to composed interface, with the +knowledge graph and real-time state engine at its centre. + +``` +SOURCE (AEMO, AIS, BOM, ENTSO-E, EIA, ...) + ↓ connectors: discover → fetch +RAW / BRONZE verbatim bytes + fetch metadata, never rewritten + ↓ connectors: parse → normalize (reproducible, versioned) +NORMALISED / SILVER Observation envelopes, append-only JSONL per feed + ↓ state engine: project / aggregate / correct +RECONCILED / GOLD current-state snapshots, hash-chained graph deltas, inferences + ↓ +KNOWLEDGE GRAPH what exists (structural) — bitemporal nodes and edges +REAL-TIME STATE what it is doing now — per (entity, metric) projections + ↓ +API REST + SSE, provenance-tagged responses, SDUI composition + ↓ +GLOBE GPU-rendered human interface (deck.gl) +``` + +## Package dependency order + +``` +ontology → registry, store → graph, state → connectors, intel → pact-bridge, api → globe +``` + +- **ontology** owns the envelopes (schemas + TS types + validators + ID grammar). Every + other package speaks these types; nothing else defines a data shape. +- **registry** loads the YAML registries (class-A structural truth) and enforces + referential integrity: every `source_id`, `entity_id`, `grid_id` mentioned anywhere + must resolve. +- **store** owns the on-disk medallion layout under `intelligence/var/` (see + `STORAGE.md`). Bronze is immutable history; silver is append-only observations; gold + is derived and always reproducible from bronze + code. +- **graph** holds the structural world model in memory, loaded from the registry, and + applies runtime deltas only through a hash-chained journal (the journal is truth, + memory is a projection — the same event-sourced stance as PACT's §6 event log). +- **state** turns observations into per-entity current state with event-time semantics, + correction handling, and window aggregation. It never mutates canonical entities. +- **connectors** implement source-specific ingestion behind a common SDK. The + `HttpGate` abstraction makes fixture replay and live fetching the same code path. +- **intel** runs deterministic, versioned rules over state + graph and emits + evidence-linked Inference records — the only producer of class-C data. +- **api** composes registry ⊕ graph ⊕ state ⊕ gaps ⊕ inferences at read time and + serves REST + SSE plus SDUI panel documents. +- **pact-bridge** publishes inferences into a PACT fabric as `fact`-type proposals and + demonstrates competing-hypotheses resolution against the reference server. +- **globe** renders it. + +## Phase mapping + +How the handoff's build phases land in this tree: + +| Phase | Landing | +|---|---| +| 1 Grid discovery | `data/registries/grids/` | +| 2–3 Source discovery + registry | `data/registries/sources/` | +| 4 Ontology | `schemas/` + `packages/ontology` | +| 5 Relationship graph | `schemas/relationship.schema.json` + `packages/graph` | +| 6 Ingestion | `packages/store` + `packages/connectors` | +| 7–9 Energy chains, electricity, industrial load | `data/entities/au-nem/`, `data/relationships/au-nem/`, AEMO connectors | +| 10 Facility telemetry classification | `observability` field + `data/registries/gaps/` | +| 11 Production | structural PRODUCES/CONSUMES edges + intel rules | +| 12 Live logistics | `aisstream` connector + vessel entities | +| 13 Product destination | SHIPS_TO / EXPORTS_TO edges at defensible level | +| 14 Real-time state engine | `packages/state` | +| 15 Real-time knowledge graph | `packages/graph` + state attachment at API read time | +| 16 AI role | `packages/intel` (deterministic rules now; AI hook documented in `SDUI.md`) | +| 17 PACT | `packages/pact-bridge` + RFC in `docs/v2-prep/` | +| 18 SDUI | `packages/api` composer + `schemas/sdui-panel.schema.json` | +| 19–20 Globe + LOD | `packages/globe` | +| 21 3DGS | documented placeholder (`GLOBE.md`) | + +## Design invariants + +1. **Class separation is structural.** Structural, telemetry, and inference data live in + different envelopes with different schemas. There is no field an inference could set + to pass as an observation. +2. **Raw is sacred.** Bronze artifacts are never rewritten or destroyed. Every + downstream transformation is reproducible: silver and gold can be deleted and + rebuilt from bronze + code at any time. +3. **Canonical entities are never mutated by telemetry.** State attaches to entities at + read time, keyed by `entity_id`. +4. **Absence is explicit.** Missing live coverage is a first-class observability-gap + record, not a silent hole. +5. **Honesty about verification.** A source is `verified_live` only after the + verification harness has actually reached it. Everything authored under blocked + egress says `documented_only`. diff --git a/intelligence/docs/DATA-MODEL.md b/intelligence/docs/DATA-MODEL.md new file mode 100644 index 0000000..d77c179 --- /dev/null +++ b/intelligence/docs/DATA-MODEL.md @@ -0,0 +1,116 @@ +# Data model + +Canonical schemas live in `../schemas/` (JSON Schema 2020-12, the same dialect as +`spec/v2.0+/schemas/`). The TypeScript mirror lives in `../packages/ontology/src/`. +This document explains the semantics the schemas encode. + +## The three data classes + +| Class | Envelope | Written by | Mutability | +|---|---|---|---| +| A. Structural | Entity, Relationship | registry YAML (curated) + gold deltas | bitemporal, versioned | +| B. Live telemetry | Observation | connectors only | append-only, corrections supersede | +| C. Derived intelligence | Inference | intel rules only | status transitions, never edited in place | + +Class is determined by which envelope a datum lives in — there is no `data_class` field +an author could set dishonestly. The entity envelope carries `data_class: "structural"` +as a JSON Schema `const` so the tag is machine-readable but not author-choosable. + +## Entity (class A) + +An entity is a durable thing in the world: a grid, a generator, a smelter, a vessel, a +port, a company, a commodity. Key fields: + +- `entity_id` — canonical ID per the grammar in `IDENTIFIERS.md`. +- `entity_type` — one of the 40 canonical types (Grid, GridRegion, Operator, Generator, + StorageAsset, TransmissionAsset, Interconnector, Substation, CoalBasin, CoalMine, + GasBasin, GasField, GasProcessingPlant, Pipeline, HydroReservoir, IndustrialFacility, + Smelter, SteelMill, Mine, Refinery, LNGPlant, ChemicalPlant, DataCentre, Port, + Terminal, Berth, Railway, Train, Vessel, Company, Commodity, Product, Market, + Destination, Sensor, TelemetryFeed, Observation, Source, Evidence, Inference, + ObservabilityGap — the final five are reified record kinds that remain addressable as + graph nodes). +- `observability` — the facility-telemetry classification from the handoff: + `KNOWN_LIVE` (a live feed observes this entity directly), `ESTIMATED` (state is + derived from proxies), `NOT_OBSERVABLE` (no lawful path to live state today), + `INSTRUMENTATION_OPPORTUNITY` (a gap record proposes owned sensing), `UNCLASSIFIED`. +- `geometry` — GeoJSON Point/LineString/Polygon/MultiPolygon in WGS84, or null. +- `external_ids` — typed cross-references (AEMO DUIDs, ENTSO-E EICs, IMO, MMSI, + UN/LOCODE, GEM IDs, ...). The key vocabulary is closed; see `IDENTIFIERS.md`. +- `properties` — per-type payload validated by `schemas/entity-types/*.properties.json`. +- `sources[]` — every structural record cites the source registry entries it came from. + +## Relationship (class A, bitemporal) + +Typed edge between two entities. The 24 relationship types: + +`PART_OF, CONNECTED_TO, INTERCONNECTED_WITH, OWNED_BY, OPERATED_BY, EXTRACTS, SUPPLIES, +MAY_SUPPLY, TRANSPORTS, GENERATES, TRANSMITS, IMPORTS_FROM, EXPORTS_TO, CONSUMES, +PRODUCES, LOADS_AT, DEPARTS_FROM, ARRIVES_AT, SHIPS_TO, OBSERVED_BY, REPORTED_BY, +INFERRED_FROM, SUPPORTED_BY, CONTRADICTED_BY` + +Two time axes: + +- **World time** — `valid_from` / `valid_to`: when the relationship held in reality + (a mine supplied a power station 2011–2019). +- **System time** — `recorded_at` / `superseded_at`: when we believed it. Corrections + never delete; they close the old record (`superseded_at`) and append a new one. + +Every relationship carries `confidence` (0–1), `source`, and +`method: declared | derived | inferred | manual`. When `method` is `inferred`, +`evidence[]` is required — an edge asserted by a rule must point at the observations +and structural facts that justify it. + +## Observation (class B) + +One measurement of one metric on one entity: + +- `observation_id` — ULID (time-ordered, collision-free at our scale). +- `metric` — dotted path from the metric vocabulary (`power.output.mw`, + `market.price.energy`, `grid.frequency.hz`, `vessel.position`, ...). +- `event_time` vs `ingest_time` — when reality happened vs when we learned of it. + All state-engine semantics key on event time; latency is the visible difference. +- `quality` — `good | suspect | estimated | interpolated | missing`. +- `is_correction` / `corrects` / `source_sequence` — publishers reissue intervals + (AEMO dispatch runs, for example). A correction supersedes the original observation + in projections but both remain on disk. + +## Inference (class C) + +The formalisation of the PACT `fact` proposal payload `{ claim, evidence, tier, +sources }` (see the RFC in `../../docs/v2-prep/`). Key fields: + +- `claim` — one human-readable sentence; `claim_structured` — subject / predicate / + object / qualifiers for machine use. +- `tier` — evidence tier: `A` (direct observation of the claimed quantity), + `B` (strong proxy or single-hop derivation), `C` (multi-hop inference or pattern + match). +- `confidence` — 0–1, produced by the rule, never hand-authored. +- `status` — `INFERRED → CORROBORATED | CONTESTED → VERIFIED | RETRACTED`. Status moves + through PACT consensus (see `packages/pact-bridge`) or new evidence; records are + never edited in place. +- `evidence[]` and `contrary_evidence[]` — references to observations, structural + records, or prior inferences, each with a role. An inference with an empty evidence + array is schema-invalid. +- `method` — `rule:{name}@{semver}`: the exact versioned rule that produced it, so + every inference is reproducible. +- `lineage` — `correlation_id` / `in_response_to` / `prev_hash`, mirroring the PACT + event envelope (`spec/v2.0/schemas/event.json`) so inference chains ride the same + audit spine as PACT events rather than inventing a parallel one. + +## Source and TelemetryFeed + +A **Source** is a publisher's dataset (registry record; see `../data/registries/sources/`). +Its `verification_status` is the honesty mechanism: `verified_live` (the harness reached +it), `documented_only` (built from published documentation), `unverified`. +A **TelemetryFeed** is one concrete stream a connector produces from a source +(e.g. `feed:aemo:dispatchis:regionsum`), the unit silver files are keyed by. + +## Observability gap + +The reification of "we cannot see this and it matters" — entity, desired metric, +best available proxy, required resolution/accuracy, commercial and strategic value, +instrumentation options (CT sensor, substation gateway, AIS receiver, SCADA/OPC-UA, +Modbus, MQTT gateway, satellite, camera, partner feed), complexity, priority, status. +Gap records drive both the API (`/api/intel/gaps`) and the globe's gap-ring layer, and +are the input to Tailor/Sovrgn owned-instrumentation decisions. diff --git a/intelligence/docs/GLOBE.md b/intelligence/docs/GLOBE.md new file mode 100644 index 0000000..dcf3ed1 --- /dev/null +++ b/intelligence/docs/GLOBE.md @@ -0,0 +1,53 @@ +# Globe + +The human interface is a GPU-rendered interactive globe (`packages/globe`), built with +deck.gl's `_GlobeView` and instanced layers. It consumes only the public API — it holds +no privileged data path — and it was deliberately built **last**, after the data +architecture, per the build-order rule. + +## Rendering decisions + +- **deck.gl over raw Three.js:** instanced Scatterplot/Arc/Path/GeoJson layers give a + globe capable of millions of instances without hand-writing projection math or + shaders; custom layers and the WebGPU path remain available as escape hatches. +- **No DOM elements for geographic objects.** Everything geographic is GPU-instanced. + DOM is reserved for the SDUI side panel and controls. +- **Bundled basemap:** simplified Natural Earth 110m coastline/country GeoJSON + (public domain, ~250 KB) is committed and served statically — the globe renders with + zero external tile or font requests, which also keeps it working under blocked + egress. + +## Layers + +| Layer | deck.gl type | Encoding | +|---|---|---| +| Grid boundaries | GeoJsonLayer | fill by live demand intensity | +| Generators | ScatterplotLayer | radius = capacity, colour = live MW / capacity | +| Interconnectors | ArcLayer | width = |flow|, animated direction | +| Vessels | IconLayer | heading-rotated, colour = cargo status (reported vs inferred) | +| Observability gaps | ScatterplotLayer (ring style) | distinct ring — gaps are a first-class visual | + +## Level of detail + +Zoom-gated layer sets in `src/lod.ts`: + +1. **Global** — grid regions, aggregate demand, major interconnector arcs, vessels in + transit on major routes. +2. **Continental** — major generation assets, basins, ports, cross-border flows. +3. **Regional** — all facilities, transmission, vessels/trains; the Gladstone demo + scene lives here. +4. **Facility** — SDUI side panel (`/api/intel/sdui/{entityId}`) with live state, + process inputs/outputs, upstream/downstream chains. + +## Live feel under replay + +The globe subscribes to `/api/intel/stream` (SSE). When the API runs in replay mode it +re-emits fixture observations on a compressed clock, so the demo visibly ticks — +described in the UI as replayed fixture data, never as live telemetry. + +## 3DGS placeholder + +The facility LOD reserves a mount point for 3D Gaussian Splatting site views +(globe → region → facility → 3DGS digital twin). Out of scope for this workspace +today; the mount contract is: given `entity_id`, the panel receives a `splat_url` +property from the entity's `properties.visualisation` block when one exists. diff --git a/intelligence/docs/IDENTIFIERS.md b/intelligence/docs/IDENTIFIERS.md new file mode 100644 index 0000000..01005db --- /dev/null +++ b/intelligence/docs/IDENTIFIERS.md @@ -0,0 +1,83 @@ +# Identifiers + +## Canonical entity IDs + +Structured stable slugs, not UUIDs — registries are YAML reviewed in pull requests, and +slugs diff, merge, and grep. Grammar (enforced by regex in the schemas and by +`packages/ontology/src/ids.ts`): + +``` +{type-code}:{namespace}:{local-slug} (or {type-code}:{namespace} for singletons) +``` + +Examples: + +``` +grid:au-nem region:au-nem:qld1 gen:au-nem:gladstone-ps +intercon:au-nem:qni sub:au-qld:calvale line:au-qld:calvale-halys +basin:au:bowen mine:au-qld:blackwater gasfield:au-qld:surat-fairview +pipe:au:qsn-link resv:au-tas:gordon smelter:au-qld:boyne-island +steel:au-nsw:port-kembla refinery:au-qld:lytton lng:au-qld:qclng +chem:au-qld:orica-yarwun dc:au-nsw:eastern-creek port:au-qld:gladstone +terminal:au-qld:rg-tanna berth:au-qld:rg-tanna-1 rail:au-qld:blackwater-system +train:au-qld:aurizon-bw-01 vessel:imo:9700000 vessel:mmsi:503123456 +company:au:rio-tinto commodity:coal-metallurgical product:aluminium-ingot +market:jp:steel dest:jp:port-of-nagoya sensor:au-qld:boyne-ct-01 +feed:aemo:dispatchis:regionsum source:aemo:nemweb-dispatchis gap:au-nem:boyne-live-load +infer:2026-08-12:rgt-cargo-01 obs: (ULIDs, not slugs — see below) +``` + +Type codes: `grid, region, operator, gen, storage, trans, intercon, sub, line, basin, +mine, gasbasin, gasfield, gasplant, pipe, resv, facility, smelter, steel, refinery, +lng, chem, dc, port, terminal, berth, rail, train, vessel, company, commodity, product, +market, dest, sensor, feed, source, evidence, infer, gap`. + +Namespace rules: + +- Grid-bound assets use the grid ID's namespace (`au-nem`). +- Geographic assets use ISO country (+ state where useful): `au-qld`, `jp`, `us-tx`. +- Where a global external authority is the natural key, it becomes the namespace: + `vessel:imo:{imo}`, falling back to `vessel:mmsi:{mmsi}` when no IMO number exists. +- Commodities and products are global: `commodity:{slug}`, `product:{slug}`. + +**Immutability:** a published ID is never renamed. A misnamed entity gets an alias and, +if truly wrong, a tombstone record pointing at the successor (`same_as`). + +Observation IDs are ULIDs (time-sortable, generated at ingest), not slugs — there are +too many and they are never hand-authored. + +## External ID vocabulary + +`external_ids` is a closed keyed object (`additionalProperties: false`): + +| Key | Authority | Example | +|---|---|---| +| `aemo_duid` | AEMO dispatchable unit ID(s) | `["GSTONE1", "GSTONE2"]` | +| `aemo_station_id` | AEMO station identifier | `GSTONE` | +| `aemo_region_id` | AEMO region | `QLD1` | +| `entsoe_eic` | ENTSO-E Energy Identification Code | `10YAU-...` | +| `eia_plant_code` | US EIA plant code | `55123` | +| `gem_id` | Global Energy Monitor tracker ID | `G100000108417` | +| `imo` | IMO ship number | `9700000` | +| `mmsi` | Maritime Mobile Service Identity | `503123456` | +| `callsign` | Vessel callsign | `VJN2` | +| `unlocode` | UN/LOCODE port code | `AUGLT` | +| `abn` | Australian Business Number | `47004458404` | +| `lei` | Legal Entity Identifier | `213800YOEO5OQ72G2R82` | +| `wikidata_qid` | Wikidata | `Q1140983` | +| `osm_id` | OpenStreetMap | `way/123456789` | + +Adding a key to this vocabulary is a schema change (deliberate, reviewed) — free-form +external references are not allowed, because entity resolution depends on typed keys. + +## Entity resolution + +`packages/registry/src/resolve.ts` runs two passes: + +1. **Exact:** identical typed external ID (same `imo`, same `aemo_duid`, ...) → + the records refer to the same entity. +2. **Heuristic:** normalised-name similarity + geometry proximity (< 5 km) → + emits `same_as` **candidates** to `var/resolution-candidates.jsonl` for human review. + +Candidates are never auto-merged. In this session's hand-curated seed data, resolution +runs as a validation step (it should find no unexpected duplicates), not as a mutation. diff --git a/intelligence/docs/LEGAL-LICENSING.md b/intelligence/docs/LEGAL-LICENSING.md new file mode 100644 index 0000000..ca62e58 --- /dev/null +++ b/intelligence/docs/LEGAL-LICENSING.md @@ -0,0 +1,53 @@ +# Legal & licensing posture + +Engineering metadata, not legal advice. Every source registry record carries a +`licensing` field summarising the publisher's terms as understood from their published +documentation; this file records the workspace-wide posture and the analyses that +shaped connector choices. + +## Ground rules + +1. **No scraping against terms of service.** Connectors are built only for sources + whose published terms permit programmatic access for this use, or which are + explicitly open data. +2. **No bulk third-party data committed to the repo.** The repo contains structural + facts (locations, capacities, identifiers — cited to their sources), small synthetic + fixtures flagged `fixture_provenance: synthetic-from-spec`, and public-domain + basemap geometry (Natural Earth). It does not contain redistributed datasets. +3. **Private contracts are never fabricated.** Product-destination edges stop at the + highest defensible level (port, country, market). Inferred commercial relationships + are `MAY_SUPPLY` with confidence and evidence, never presented as contracts. +4. **Attribution obligations** recorded per source in its registry record and honoured + in API responses via the `sources[]` provenance blocks. + +## Analyses + +- **AIS.** Design target is aisstream.io, whose published terms permit free + programmatic streaming access with an API key for non-redistribution use. Scraping + commercial AIS aggregator websites (e.g. MarineTraffic) is rejected — their terms + prohibit it. Commercial feeds (Spire, exactEarth, or a direct AISHub membership) are + the documented upgrade path for production coverage; owned AIS receivers are an + instrumentation option recorded in the gap registry. AIS itself is an open VHF + broadcast intended for navigation safety; receiving and decoding it is lawful in + Australia, but redistribution terms attach to third-party *aggregated* feeds. +- **AEMO.** NEMWEB and the MMS data model are published for public use with + attribution; AEMO's terms permit reproduction for study/analysis. Registration data + and dashboards are public. Connector cadence respects published file cadence + (5-minute dispatch); no authentication is bypassed anywhere. +- **BOM.** Observation products are publicly published; BOM's copyright notice permits + use with attribution. High-volume programmatic access has a registered/paid channel + (documented in the source record) — the connector is designed for low-cadence + station observations. +- **ENTSO-E Transparency.** Free but registration-required (security token). The + connector is a documented stub; no token, no fetch. +- **EIA.** US federal open data, API key freely issued; public domain. +- **Natural Earth.** Public domain; committed. +- **Global Energy Monitor.** CC BY 4.0 for most trackers — usable with attribution; + recorded per record where GEM is a source. + +## Committed-data provenance + +Structural seed records in `../data/entities/` cite their sources in each record's +`sources[]` array. Facts of the world (a smelter's location, a generator's DUID and +capacity) are not copyrightable as facts, but the compilations they were read from are +credited, and nothing is bulk-copied. diff --git a/intelligence/docs/SDUI.md b/intelligence/docs/SDUI.md new file mode 100644 index 0000000..8f5649d --- /dev/null +++ b/intelligence/docs/SDUI.md @@ -0,0 +1,63 @@ +# Server-driven UI (SDUI) + +The interface for an entity is not a fixed dashboard: the server composes a panel +document from the entity's type and its live situation, and the client renders it. +Panel documents validate against `../schemas/sdui-panel.schema.json`. + +## Panel document + +```json +{ + "entity_id": "smelter:au-qld:boyne-island", + "entity_type": "Smelter", + "situation": ["has-gaps", "estimated-load"], + "layout": [ + { "panel": "headline-state", + "data": { "endpoint": "/api/intel/entities/smelter:au-qld:boyne-island/state" } }, + { "panel": "timeseries", + "props": { "metric": "power.load.estimated_mw", "window": "24h" }, + "data": { "endpoint": "/api/intel/entities/smelter:au-qld:boyne-island/observations?metric=power.load.estimated_mw" } }, + { "panel": "gap-card", + "data": { "endpoint": "/api/intel/gaps?entity=smelter:au-qld:boyne-island" } }, + { "panel": "graph-neighborhood", "props": { "depth": 2 }, + "data": { "endpoint": "/api/intel/entities/smelter:au-qld:boyne-island/graph?depth=2" } }, + { "panel": "inference-list", + "data": { "endpoint": "/api/intel/inferences?entity=smelter:au-qld:boyne-island" } } + ] +} +``` + +Panels reference **endpoints**, never inline data values — the client fetches, so the +data a user sees is always the API's provenance-tagged truth, not a copy baked into a +layout. + +## Composition + +`packages/api/src/sdui/composer.ts`: + +1. **Type template** — each entity type has a deterministic base template + (`templates/grid.ts`, `generator.ts`, `smelter.ts`, `vessel.ts`, `port.ts`, + `default.ts`): a grid leads with load/generation/price/flows; a generator with + current MW vs capacity and its upstream fuel chain; a vessel with position, speed, + cargo status (reported vs inferred, visibly distinct), origin and destination. +2. **Situation rules** — computed from live state at composition time: + `no-live-data`, `stale-telemetry` (staleness beyond the feed's expected cadence), + `has-gaps`, `estimated-load` (observability = ESTIMATED), `anomaly-active` (an + open anomaly inference references the entity). Situations inject or reorder panels — + an active anomaly hoists the inference list; missing live data hoists the gap card. + +## AI hook + +`composeWithAI(entity, situations, deterministicLayout)` is the declared seam for +model-driven composition. Contract: + +- AI may **reorder, group, annotate, or drop** panels, and may add panels that + reference *declared* endpoints. +- AI may **not** inline data values, invent endpoints, or alter the `data` blocks of + existing panels. Composed output is schema-validated and endpoint-checked before + serving; violations fall back to the deterministic layout. +- The current implementation is the identity function — deterministic output ships + unchanged. The hook exists so the AI layer lands without renegotiating the contract. + +This is the SDUI expression of the system-wide rule: **AI interprets and arranges; +it does not manufacture state.** diff --git a/intelligence/docs/STORAGE.md b/intelligence/docs/STORAGE.md new file mode 100644 index 0000000..cde63ea --- /dev/null +++ b/intelligence/docs/STORAGE.md @@ -0,0 +1,49 @@ +# Storage + +## Embedded default (this repo) + +Zero external services: everything runs in-process against the filesystem, under the +gitignored `intelligence/var/`. This is deliberate — CI, the smoke test, and a laptop +all get the identical stack, and the layout is the *contract*; the engine behind it is +swappable. + +``` +var/ + bronze/{source_id}/{yyyy}/{mm}/{dd}/{artifact_id}.bin verbatim fetched bytes + bronze/{source_id}/{yyyy}/{mm}/{dd}/{artifact_id}.meta.json url, fetched_at, sha256, + http_status, headers subset + silver/{feed_id}/{yyyy-mm-dd}.jsonl one Observation per line, + append-only + gold/state/{snapshot_ts}.json current-state snapshots + gold/graph/deltas.jsonl entity/relationship patches, + hash-chained (prev_hash) + gold/inferences/{yyyy-mm-dd}.jsonl Inference records + resolution-candidates.jsonl entity-resolution output +``` + +Rules: + +- **Bronze is immutable.** Artifacts are content-addressed by sha256 in their metadata; + a re-fetch of changed content is a new artifact, never an overwrite. +- **Silver is append-only.** Corrections are new lines with `is_correction: true`; + nothing is rewritten. +- **Gold is disposable.** Anything in gold must be reproducible from bronze + versioned + code. Deleting `var/gold` and re-running ingestion is always safe. +- **The graph delta journal is truth.** `gold/graph/deltas.jsonl` is hash-chained + (`prev_hash` over the canonical JSON of the previous line, `"GENESIS"` first — the + same construction as PACT §6.4). In-memory graph state is a projection of registry + YAML + this journal. + +## Production mapping (documented, not built here) + +| Embedded piece | Production substitution | +|---|---| +| bronze files | object store (S3/GCS) with lifecycle rules, same path scheme | +| silver JSONL | TimescaleDB hypertables or ClickHouse (observations table, partitioned by feed/day) | +| current-state projection | Redis/inline materialised view fed by the same delta bus | +| gold graph journal | the same journal, plus a graph database (Kùzu embedded or Neo4j) built from it | +| cooperative scheduler | per-connector workers (containers) + a queue; identical Connector interface | + +The `Store` interface in `packages/store/src/` is the seam: implementations must +preserve the three-tier rules above, and nothing outside the package may touch `var/` +directly. diff --git a/intelligence/package-lock.json b/intelligence/package-lock.json new file mode 100644 index 0000000..e41498b --- /dev/null +++ b/intelligence/package-lock.json @@ -0,0 +1,5300 @@ +{ + "name": "@pact-tailor/intelligence-workspace", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@pact-tailor/intelligence-workspace", + "version": "0.1.0", + "workspaces": [ + "packages/*" + ], + "devDependencies": { + "@types/node": "^22.10.0", + "typescript": "^5.7.2" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@amcharts/amcharts5": { + "version": "5.14.4", + "resolved": "https://registry.npmjs.org/@amcharts/amcharts5/-/amcharts5-5.14.4.tgz", + "integrity": "sha512-Tl7wQLWvsvyWVtlCIm1yhZtJviSDYjuNTnlUkO0D49GyByoK0nb9fr0DK4rUw4DVgyLcySxWBsb2lzTJm5Rd9Q==", + "license": "SEE LICENSE IN LICENSE", + "peer": true, + "dependencies": { + "@types/d3": "^7.0.0", + "@types/d3-chord": "^3.0.0", + "@types/d3-hierarchy": "3.1.1", + "@types/d3-sankey": "^0.11.1", + "@types/d3-shape": "^3.0.0", + "@types/geojson": "^7946.0.8", + "@types/polylabel": "^1.0.5", + "@types/svg-arc-to-cubic-bezier": "^3.2.0", + "d3": "^7.0.0", + "d3-chord": "^3.0.0", + "d3-force": "^3.0.0", + "d3-geo": "^3.0.0", + "d3-hierarchy": "^3.0.0", + "d3-sankey": "^0.12.3", + "d3-selection": "^3.0.0", + "d3-shape": "^3.0.0", + "d3-transition": "^3.0.0", + "d3-voronoi-treemap": "^1.1.2", + "flatpickr": "^4.6.13", + "markerjs2": "^2.29.4", + "pdfmake": "^0.2.2", + "polylabel": "^1.1.0", + "seedrandom": "^3.0.5", + "svg-arc-to-cubic-bezier": "^3.2.0", + "tslib": "^2.2.0" + } + }, + "node_modules/@arcgis/core": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@arcgis/core/-/core-4.34.8.tgz", + "integrity": "sha512-UrEBTjXpSA9fhmmnAENBzz9GG81xALTezQFMXUs2iMB+tiOckmJyBbhATI/W4lIQyUfNEK7Zm/46EP2PhDga/A==", + "license": "SEE LICENSE IN copyright.txt", + "peer": true, + "dependencies": { + "@amcharts/amcharts5": "~5.14.1", + "@arcgis/toolkit": "^4.34.0", + "@esri/arcgis-html-sanitizer": "~4.1.0", + "@esri/calcite-components": "^3.3.2", + "@vaadin/grid": "~24.9.1", + "@zip.js/zip.js": "~2.8.7", + "luxon": "~3.7.2", + "marked": "~16.3.0", + "tslib": "^2.8.1" + } + }, + "node_modules/@arcgis/lumina": { + "version": "4.34.9", + "resolved": "https://registry.npmjs.org/@arcgis/lumina/-/lumina-4.34.9.tgz", + "integrity": "sha512-efqO+SwR+1IYf29AATh1l2FUeypRyRINTBNkaJY+KkaFe+8gqSJ45qOmputhyzF5WTRDb7WhOYgnChjp6VYPpA==", + "license": "SEE LICENSE IN LICENSE.md", + "peer": true, + "dependencies": { + "@arcgis/toolkit": "~4.34.9", + "csstype": "^3.1.3", + "tslib": "^2.8.1" + }, + "peerDependencies": { + "@lit/context": "^1.1.5", + "lit": "^3.3.0" + }, + "peerDependenciesMeta": { + "@lit/context": { + "optional": true + } + } + }, + "node_modules/@arcgis/toolkit": { + "version": "4.34.9", + "resolved": "https://registry.npmjs.org/@arcgis/toolkit/-/toolkit-4.34.9.tgz", + "integrity": "sha512-wFST+eVnCwmg9NyICVyn9bsBnR+TlWklsGqG3L7xqSTgfXo6TuCThE7wtTb8xWxsTBkGvImqMUgpgLuwQuTQ1g==", + "license": "SEE LICENSE IN LICENSE.md", + "peer": true, + "dependencies": { + "tslib": "^2.8.1" + } + }, + "node_modules/@carto/api-client": { + "version": "0.5.32", + "resolved": "https://registry.npmjs.org/@carto/api-client/-/api-client-0.5.32.tgz", + "integrity": "sha512-MoaBall43ehEox1aVoMd07UvoW7FtFtB0PV086gqFzoYgs3BS19Yl59zSL9g2oAvXoG4P1ckXHAnCkfIeTDYog==", + "license": "MIT", + "dependencies": { + "@loaders.gl/schema": "^4.3.3", + "@types/geojson": "^7946.0.16", + "d3-format": "^3.1.2", + "d3-scale": "^4.0.2", + "h3-js": "^4.1.0", + "jsep": "^1.4.0", + "quadbin": "^0.4.1-alpha.0" + } + }, + "node_modules/@deck.gl/aggregation-layers": { + "version": "9.3.10", + "resolved": "https://registry.npmjs.org/@deck.gl/aggregation-layers/-/aggregation-layers-9.3.10.tgz", + "integrity": "sha512-Fnb/Azwq9CHSP5sBgAewRbRTmHNrFiQEFZShlreEkzXDkv+nl8DJo2OzU51Uxo+m0JMO+bwYFNgQFV4Nc3DtfQ==", + "license": "MIT", + "dependencies": { + "@luma.gl/shadertools": "^9.3.5", + "@math.gl/core": "^4.1.0", + "@math.gl/web-mercator": "^4.1.0", + "d3-hexbin": "^0.2.1" + }, + "peerDependencies": { + "@deck.gl/core": "~9.3.0", + "@deck.gl/layers": "~9.3.0", + "@luma.gl/core": "~9.3.5", + "@luma.gl/engine": "~9.3.5" + } + }, + "node_modules/@deck.gl/arcgis": { + "version": "9.3.10", + "resolved": "https://registry.npmjs.org/@deck.gl/arcgis/-/arcgis-9.3.10.tgz", + "integrity": "sha512-8XT1HWONOC6hA5V1kxNHiCGUeZYRexz8WY4KG7YWNIfM+yumqfXF1nJFd8DFLBWrv8yVyMeREX1VvLgBGniKFA==", + "license": "MIT", + "dependencies": { + "esri-loader": "^3.7.0" + }, + "peerDependencies": { + "@arcgis/core": "^4.0.0", + "@deck.gl/core": "~9.3.0", + "@luma.gl/core": "~9.3.5", + "@luma.gl/engine": "~9.3.5", + "@luma.gl/webgl": "~9.3.5" + } + }, + "node_modules/@deck.gl/carto": { + "version": "9.3.10", + "resolved": "https://registry.npmjs.org/@deck.gl/carto/-/carto-9.3.10.tgz", + "integrity": "sha512-YMXnScpqP2rXfzjrwn19l8HFZathH16e9YkZ8ZxyPNUkfC6CC67V/+NQGwQuWGodhChAEgpKjR3kofSirTGQAg==", + "license": "MIT", + "dependencies": { + "@carto/api-client": "^0.5.19", + "@loaders.gl/compression": "^4.4.3", + "@loaders.gl/gis": "^4.4.3", + "@loaders.gl/loader-utils": "^4.4.3", + "@loaders.gl/mvt": "^4.4.3", + "@loaders.gl/schema": "^4.4.3", + "@loaders.gl/tiles": "^4.4.3", + "@luma.gl/core": "^9.3.5", + "@luma.gl/shadertools": "^9.3.5", + "@math.gl/web-mercator": "^4.1.0", + "@types/d3-array": "^3.0.2", + "@types/d3-color": "^3.1.3", + "@types/d3-scale": "^3.0.0", + "cartocolor": "^5.0.2", + "d3-array": "^3.2.0", + "d3-color": "^3.1.0", + "d3-format": "^3.1.0", + "d3-scale": "^4.0.0", + "earcut": "^2.2.4", + "h3-js": "^4.4.0", + "moment-timezone": "^0.6.0", + "pbf": "^3.2.1", + "quadbin": "^0.4.0" + }, + "peerDependencies": { + "@deck.gl/aggregation-layers": "~9.3.0", + "@deck.gl/core": "~9.3.0", + "@deck.gl/extensions": "~9.3.0", + "@deck.gl/geo-layers": "~9.3.0", + "@deck.gl/layers": "~9.3.0", + "@loaders.gl/core": "^4.4.3", + "@luma.gl/core": "~9.3.5" + } + }, + "node_modules/@deck.gl/core": { + "version": "9.3.10", + "resolved": "https://registry.npmjs.org/@deck.gl/core/-/core-9.3.10.tgz", + "integrity": "sha512-BAJ6r+eRV4euuGcIi2SnlzrodgZ+sD2VuXsoL1DER0LwwKXmKmQu6WA8ax6DB9KFsBghLVoTUN5zaCGF6uKHuQ==", + "license": "MIT", + "dependencies": { + "@loaders.gl/core": "^4.4.3", + "@loaders.gl/images": "^4.4.3", + "@luma.gl/core": "^9.3.5", + "@luma.gl/engine": "^9.3.5", + "@luma.gl/shadertools": "^9.3.5", + "@luma.gl/webgl": "^9.3.5", + "@math.gl/core": "^4.1.0", + "@math.gl/sun": "^4.1.0", + "@math.gl/types": "^4.1.0", + "@math.gl/web-mercator": "^4.1.0", + "@probe.gl/env": "^4.1.1", + "@probe.gl/log": "^4.1.1", + "@probe.gl/stats": "^4.1.1", + "@types/offscreencanvas": "^2019.6.4", + "gl-matrix": "^3.0.0", + "mjolnir.js": "^3.0.0" + } + }, + "node_modules/@deck.gl/extensions": { + "version": "9.3.10", + "resolved": "https://registry.npmjs.org/@deck.gl/extensions/-/extensions-9.3.10.tgz", + "integrity": "sha512-VT5NxR+SgUCwvaWeKFKkql5HV90KjyiPyXYlzKfxxXoE8wxlOgt5Enl2MTAOLcIhlCymLp9vtWO/HEgTfvlzDw==", + "license": "MIT", + "dependencies": { + "@luma.gl/shadertools": "^9.3.5", + "@luma.gl/webgl": "^9.3.5", + "@math.gl/core": "^4.1.0" + }, + "peerDependencies": { + "@deck.gl/core": "~9.3.0", + "@luma.gl/core": "~9.3.5", + "@luma.gl/engine": "~9.3.5" + } + }, + "node_modules/@deck.gl/geo-layers": { + "version": "9.3.10", + "resolved": "https://registry.npmjs.org/@deck.gl/geo-layers/-/geo-layers-9.3.10.tgz", + "integrity": "sha512-MH6/MyWTiJ02yxrkMz6bMP1Z8d13JEmxogwZmWx2I9XLDNpCnLxnXElnVxxvDvhIbWdWq/i/xl8Kn9WOLiywBg==", + "license": "MIT", + "dependencies": { + "@loaders.gl/3d-tiles": "^4.4.3", + "@loaders.gl/gis": "^4.4.3", + "@loaders.gl/loader-utils": "^4.4.3", + "@loaders.gl/mvt": "^4.4.3", + "@loaders.gl/schema": "^4.4.3", + "@loaders.gl/terrain": "^4.4.3", + "@loaders.gl/tiles": "^4.4.3", + "@loaders.gl/wms": "^4.4.3", + "@luma.gl/gltf": "^9.3.5", + "@luma.gl/shadertools": "^9.3.5", + "@math.gl/core": "^4.1.0", + "@math.gl/culling": "^4.1.0", + "@math.gl/web-mercator": "^4.1.0", + "@types/geojson": "^7946.0.8", + "a5-js": "^0.7.2", + "h3-js": "^4.4.0", + "long": "^3.2.0" + }, + "peerDependencies": { + "@deck.gl/core": "~9.3.0", + "@deck.gl/extensions": "~9.3.0", + "@deck.gl/layers": "~9.3.0", + "@deck.gl/mesh-layers": "~9.3.0", + "@loaders.gl/core": "^4.4.3", + "@luma.gl/core": "~9.3.5", + "@luma.gl/engine": "~9.3.5" + } + }, + "node_modules/@deck.gl/google-maps": { + "version": "9.3.10", + "resolved": "https://registry.npmjs.org/@deck.gl/google-maps/-/google-maps-9.3.10.tgz", + "integrity": "sha512-S84742VZ0SAdYGNVp4C4qHHeuREFLoUZHc9AP3boC69lgosOrHGHNK35zRr2sFYJNhJBLNTiTmPAHdUjIZ4yHw==", + "license": "MIT", + "dependencies": { + "@luma.gl/webgl": "^9.3.5", + "@math.gl/core": "^4.1.0", + "@types/google.maps": "^3.48.6" + }, + "peerDependencies": { + "@deck.gl/core": "~9.3.0", + "@luma.gl/core": "~9.3.5", + "@luma.gl/webgl": "~9.3.5" + } + }, + "node_modules/@deck.gl/json": { + "version": "9.3.10", + "resolved": "https://registry.npmjs.org/@deck.gl/json/-/json-9.3.10.tgz", + "integrity": "sha512-MhKcaaGVfpnPlHwLDp1o0MHCcq+JW5SOR4f5ZYriSHKKVG1gK1+xvI/2eos4Et2sLAtuS3r4OwaXHw8AjWnEEg==", + "license": "MIT", + "dependencies": { + "jsep": "^0.3.0" + }, + "peerDependencies": { + "@deck.gl/core": "~9.3.0" + } + }, + "node_modules/@deck.gl/json/node_modules/jsep": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/jsep/-/jsep-0.3.5.tgz", + "integrity": "sha512-AoRLBDc6JNnKjNcmonituEABS5bcfqDhQAWWXNTFrqu6nVXBpBAGfcoTGZMFlIrh9FjmE1CQyX9CTNwZrXMMDA==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/@deck.gl/layers": { + "version": "9.3.10", + "resolved": "https://registry.npmjs.org/@deck.gl/layers/-/layers-9.3.10.tgz", + "integrity": "sha512-F9CRJlRWXvTKlgjKmmTotm0cA5V7I1zMtdnwJbjBZTyCJ4skuskbIIt6f+BSQ+rSxM2uW4q5/8VkcCJ2SePhXg==", + "license": "MIT", + "dependencies": { + "@loaders.gl/images": "^4.4.3", + "@loaders.gl/schema": "^4.4.3", + "@luma.gl/shadertools": "^9.3.5", + "@mapbox/tiny-sdf": "^2.0.5", + "@math.gl/core": "^4.1.0", + "@math.gl/polygon": "^4.1.0", + "@math.gl/web-mercator": "^4.1.0", + "@types/geojson": "^7946.0.16", + "earcut": "^2.2.4" + }, + "peerDependencies": { + "@deck.gl/core": "~9.3.0", + "@loaders.gl/core": "^4.4.3", + "@luma.gl/core": "~9.3.5", + "@luma.gl/engine": "~9.3.5" + } + }, + "node_modules/@deck.gl/mapbox": { + "version": "9.3.10", + "resolved": "https://registry.npmjs.org/@deck.gl/mapbox/-/mapbox-9.3.10.tgz", + "integrity": "sha512-go5+imfjdq/+P5ELMX88ncuzNSpUpL4wolV7LbegwXnKcBIgU13IKbCxJnoGCSW8oxHZu+J99sGDcDyvkYtZcw==", + "license": "MIT", + "dependencies": { + "@math.gl/web-mercator": "^4.1.0" + }, + "peerDependencies": { + "@deck.gl/core": "~9.3.0", + "@luma.gl/core": "~9.3.5", + "@math.gl/web-mercator": "^4.1.0" + } + }, + "node_modules/@deck.gl/mesh-layers": { + "version": "9.3.10", + "resolved": "https://registry.npmjs.org/@deck.gl/mesh-layers/-/mesh-layers-9.3.10.tgz", + "integrity": "sha512-HzExbHWQ05fU7YgXcCQPHjHKoMc2PTWUTu3THzdXc5HXhGqpiAd/X1YA7McA9fRJBjw1yD36otMwGEXqUOEKAA==", + "license": "MIT", + "dependencies": { + "@loaders.gl/gltf": "^4.4.3", + "@loaders.gl/schema": "^4.4.3", + "@luma.gl/gltf": "^9.3.5", + "@luma.gl/shadertools": "^9.3.5" + }, + "peerDependencies": { + "@deck.gl/core": "~9.3.0", + "@luma.gl/core": "~9.3.5", + "@luma.gl/engine": "~9.3.5", + "@luma.gl/gltf": "~9.3.5", + "@luma.gl/shadertools": "~9.3.5" + } + }, + "node_modules/@deck.gl/react": { + "version": "9.3.10", + "resolved": "https://registry.npmjs.org/@deck.gl/react/-/react-9.3.10.tgz", + "integrity": "sha512-kbfceQ6uZxFS5DVwM3JLY1gSVfx+IJ8ihnkSomxX5Nhhfos1GWnH5wJo8wznKGl9WdLd/iY5bHc8q9i0f5TVjA==", + "license": "MIT", + "peerDependencies": { + "@deck.gl/core": "~9.3.0", + "@deck.gl/widgets": "~9.3.0", + "react": ">=16.3.0", + "react-dom": ">=16.3.0" + } + }, + "node_modules/@deck.gl/widgets": { + "version": "9.3.10", + "resolved": "https://registry.npmjs.org/@deck.gl/widgets/-/widgets-9.3.10.tgz", + "integrity": "sha512-eEQuxQeqIkm04d9MgUH0MtGTmW3xSvqS/nkyIGPhaoAj6VWGu850JfXVdQbnORxFweoQWPI1jfU0Q8j8twhkEA==", + "license": "MIT", + "dependencies": { + "@floating-ui/dom": "^1.7.5", + "preact": "^10.17.0" + }, + "peerDependencies": { + "@deck.gl/core": "~9.3.0", + "@luma.gl/core": "~9.3.5" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esri/arcgis-html-sanitizer": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@esri/arcgis-html-sanitizer/-/arcgis-html-sanitizer-4.1.0.tgz", + "integrity": "sha512-einEveDJ/k1180NOp78PB/4Hje9eBy3dyOGLLtLn6bSkizpUfCwuYBIXOA7Y3F/k/BsTQXgKqUVwQ0eiscWMdA==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "xss": "1.0.13" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@esri/calcite-components": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@esri/calcite-components/-/calcite-components-3.3.3.tgz", + "integrity": "sha512-tw+EfJ3pb+Odj71W6E9GUkm8rMbNxfW1KeiI8GgsKDzhr39hMKwY+zYYFFYuO0FONxWGvAB+B8yqB0NvH7WeHw==", + "license": "SEE LICENSE.md", + "peer": true, + "dependencies": { + "@arcgis/lumina": ">=4.34.0-next.158 <4.35.0", + "@arcgis/toolkit": ">=4.34.0-next.158 <4.35.0", + "@esri/calcite-ui-icons": "4.3.0", + "@floating-ui/dom": "^1.6.12", + "@floating-ui/utils": "^0.2.8", + "@types/sortablejs": "^1.15.8", + "color": "^5.0.0", + "composed-offset-position": "^0.0.6", + "es-toolkit": "^1.39.8", + "focus-trap": "^7.6.5", + "interactjs": "^1.10.27", + "lit": "^3.3.0", + "sortablejs": "^1.15.6", + "timezone-groups": "^0.10.4", + "type-fest": "^4.30.1" + } + }, + "node_modules/@esri/calcite-ui-icons": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@esri/calcite-ui-icons/-/calcite-ui-icons-4.3.0.tgz", + "integrity": "sha512-iOOuRurpjFxFVw6+aXW2JpSkRBrdOpBcbdibfPOmSPqMd1aoHBtYmYXetKoH9vfrXoBiPyO2PkDnczhsu/N9IA==", + "license": "SEE LICENSE.md", + "peer": true, + "bin": { + "spriter": "bin/spriter.js" + } + }, + "node_modules/@floating-ui/core": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.8.0.tgz", + "integrity": "sha512-0CIZ5itps/8x7BG8dEIhs53BvCUH2PCoogtakwRTut+Arm58sJooJ0AuZhLw2HJYIR5cMLNPBSS728sPho2khQ==", + "license": "MIT", + "dependencies": { + "@floating-ui/utils": "^0.2.12" + } + }, + "node_modules/@floating-ui/dom": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.8.0.tgz", + "integrity": "sha512-yXSrzeHZBTZadLOlfyhCkJHNeLJnHRnRInwdZ40L7ZiaAtrBwoYlsDrX3v5zB1Utk7CLfzcOVnVVWoXEky7Ceg==", + "license": "MIT", + "dependencies": { + "@floating-ui/core": "^1.8.0", + "@floating-ui/utils": "^0.2.12" + } + }, + "node_modules/@floating-ui/utils": { + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.12.tgz", + "integrity": "sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==", + "license": "MIT" + }, + "node_modules/@foliojs-fork/fontkit": { + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/@foliojs-fork/fontkit/-/fontkit-1.9.2.tgz", + "integrity": "sha512-IfB5EiIb+GZk+77TRB86AHroVaqfq8JRFlUbz0WEwsInyCG0epX2tCPOy+UfaWPju30DeVoUAXfzWXmhn753KA==", + "license": "MIT", + "peer": true, + "dependencies": { + "@foliojs-fork/restructure": "^2.0.2", + "brotli": "^1.2.0", + "clone": "^1.0.4", + "deep-equal": "^1.0.0", + "dfa": "^1.2.0", + "tiny-inflate": "^1.0.2", + "unicode-properties": "^1.2.2", + "unicode-trie": "^2.0.0" + } + }, + "node_modules/@foliojs-fork/linebreak": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@foliojs-fork/linebreak/-/linebreak-1.1.2.tgz", + "integrity": "sha512-ZPohpxxbuKNE0l/5iBJnOAfUaMACwvUIKCvqtWGKIMv1lPYoNjYXRfhi9FeeV9McBkBLxsMFWTVVhHJA8cyzvg==", + "license": "MIT", + "peer": true, + "dependencies": { + "base64-js": "1.3.1", + "unicode-trie": "^2.0.0" + } + }, + "node_modules/@foliojs-fork/linebreak/node_modules/base64-js": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==", + "license": "MIT", + "peer": true + }, + "node_modules/@foliojs-fork/pdfkit": { + "version": "0.15.3", + "resolved": "https://registry.npmjs.org/@foliojs-fork/pdfkit/-/pdfkit-0.15.3.tgz", + "integrity": "sha512-Obc0Wmy3bm7BINFVvPhcl2rnSSK61DQrlHU8aXnAqDk9LCjWdUOPwhgD8Ywz5VtuFjRxmVOM/kQ/XLIBjDvltw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@foliojs-fork/fontkit": "^1.9.2", + "@foliojs-fork/linebreak": "^1.1.1", + "crypto-js": "^4.2.0", + "jpeg-exif": "^1.1.4", + "png-js": "^1.0.0" + } + }, + "node_modules/@foliojs-fork/restructure": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@foliojs-fork/restructure/-/restructure-2.0.2.tgz", + "integrity": "sha512-59SgoZ3EXbkfSX7b63tsou/SDGzwUEK6MuB5sKqgVK1/XE0fxmpsOb9DQI8LXW3KfGnAjImCGhhEb7uPPAUVNA==", + "license": "MIT", + "peer": true + }, + "node_modules/@interactjs/types": { + "version": "1.10.28", + "resolved": "https://registry.npmjs.org/@interactjs/types/-/types-1.10.28.tgz", + "integrity": "sha512-vPmu4HWmsg0Fub3BPKGk7DuozkhgVK84TVkziY47v8Uh0A14gph55/vVRaQN4oZov0lGNJNK6Jyddim2qd/4vw==", + "license": "MIT", + "peer": true + }, + "node_modules/@lit-labs/ssr-dom-shim": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@lit-labs/ssr-dom-shim/-/ssr-dom-shim-1.6.0.tgz", + "integrity": "sha512-VHb0ALPMTlgKjM6yIxxoQNnpKyUKLD04VzeQdsiXkMqkvYlAHxq9glGLmgbb889/1GsohSOAjvQYoiBppXFqrQ==", + "license": "BSD-3-Clause", + "peer": true + }, + "node_modules/@lit/reactive-element": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@lit/reactive-element/-/reactive-element-2.1.2.tgz", + "integrity": "sha512-pbCDiVMnne1lYUIaYNN5wrwQXDtHaYtg7YEFPeW+hws6U47WeFvISGUWekPGKWOP1ygrs0ef0o1VJMk1exos5A==", + "license": "BSD-3-Clause", + "peer": true, + "dependencies": { + "@lit-labs/ssr-dom-shim": "^1.5.0" + } + }, + "node_modules/@loaders.gl/3d-tiles": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/3d-tiles/-/3d-tiles-4.4.4.tgz", + "integrity": "sha512-/hQnsv0N7bZQ/fr7W2F84fkxj+rE7m76HvN3DRiirIMpS2AF9nxVTSQQIxr34AyM7qTFhldqGnWHS2NMcK2e5w==", + "license": "MIT", + "dependencies": { + "@loaders.gl/compression": "4.4.4", + "@loaders.gl/crypto": "4.4.4", + "@loaders.gl/draco": "4.4.4", + "@loaders.gl/gltf": "4.4.4", + "@loaders.gl/images": "4.4.4", + "@loaders.gl/loader-utils": "4.4.4", + "@loaders.gl/math": "4.4.4", + "@loaders.gl/tiles": "4.4.4", + "@loaders.gl/zip": "4.4.4", + "@math.gl/core": "^4.1.0", + "@math.gl/culling": "^4.1.0", + "@math.gl/geospatial": "^4.1.0", + "@probe.gl/log": "^4.1.1", + "long": "^5.2.1" + }, + "peerDependencies": { + "@loaders.gl/core": "~4.4.0" + } + }, + "node_modules/@loaders.gl/3d-tiles/node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/@loaders.gl/compression": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/compression/-/compression-4.4.4.tgz", + "integrity": "sha512-8b95xmGzOv5++ehKAXwNgrSuQ6r8twTadgO3UEk8I0rTWchgYVw5klOoGjsATzjWuv4LpvXAiveSeZUs5hBLMQ==", + "license": "MIT", + "dependencies": { + "@loaders.gl/loader-utils": "4.4.4", + "@loaders.gl/worker-utils": "4.4.4", + "@types/pako": "^1.0.1", + "fflate": "0.7.4", + "pako": "1.0.11", + "snappyjs": "^0.6.1" + }, + "optionalDependencies": { + "@types/brotli": "^1.3.0", + "brotli": "^1.3.2", + "lz4js": "^0.2.0", + "zstd-codec": "^0.1" + }, + "peerDependencies": { + "@loaders.gl/core": "~4.4.0" + } + }, + "node_modules/@loaders.gl/compression/node_modules/fflate": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.7.4.tgz", + "integrity": "sha512-5u2V/CDW15QM1XbbgS+0DfPxVB+jUKhWEKuuFuHncbk3tEEqzmoXL+2KyOFuKGqOnmdIy0/davWF1CkuwtibCw==", + "license": "MIT" + }, + "node_modules/@loaders.gl/core": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/core/-/core-4.4.4.tgz", + "integrity": "sha512-uUZLs1N1VQXj3jC81LS7OkrWw1NRQHb0rdGUZI/FOFvirFEIR+TAKcvMO1Zz9WQj2dT9r6HhVjMDJQxU42LsoA==", + "license": "MIT", + "dependencies": { + "@loaders.gl/loader-utils": "4.4.4", + "@loaders.gl/schema": "4.4.4", + "@loaders.gl/schema-utils": "4.4.4", + "@loaders.gl/worker-utils": "4.4.4", + "@probe.gl/log": "^4.1.1" + } + }, + "node_modules/@loaders.gl/crypto": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/crypto/-/crypto-4.4.4.tgz", + "integrity": "sha512-KTtK2p0i50N1pJ00cAMv7ahF8jLhUu2S6tZoLFlPT3kKFuFgLbeowpw/8pC9sDBKkROCblxszLamLywjo0JXBA==", + "license": "MIT", + "dependencies": { + "@loaders.gl/loader-utils": "4.4.4", + "@loaders.gl/worker-utils": "4.4.4", + "@types/crypto-js": "^4.0.2" + }, + "peerDependencies": { + "@loaders.gl/core": "~4.4.0" + } + }, + "node_modules/@loaders.gl/draco": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/draco/-/draco-4.4.4.tgz", + "integrity": "sha512-r34fr0dzdOinANYu5ElnaJdOkk0WkinsiaU/E+MAah37x3CsU+erppDdXdkouGXqvFNbjlS9UMB1Qn5JLj3ZDg==", + "license": "MIT", + "dependencies": { + "@loaders.gl/loader-utils": "4.4.4", + "@loaders.gl/schema": "4.4.4", + "@loaders.gl/schema-utils": "4.4.4", + "@loaders.gl/worker-utils": "4.4.4", + "draco3d": "1.5.7" + }, + "peerDependencies": { + "@loaders.gl/core": "~4.4.0" + } + }, + "node_modules/@loaders.gl/geoarrow": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/geoarrow/-/geoarrow-4.4.4.tgz", + "integrity": "sha512-5IygnXr1J+IBYKA+ZEOytifS4skx/L9Rk6J/OVmNzV5gVsKQtxwA3b69GDPD61dcNAYCegA3Y4wpHO+JEqLzrg==", + "license": "MIT", + "dependencies": { + "@math.gl/polygon": "^4.1.0", + "apache-arrow": ">= 17.0.0" + }, + "peerDependencies": { + "@loaders.gl/core": "~4.4.0" + } + }, + "node_modules/@loaders.gl/gis": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/gis/-/gis-4.4.4.tgz", + "integrity": "sha512-KUonvWVntPhZKx4jjoGezXLp4R1RgS6n4FMLdOYX1fUJZ48fRDQVA28Y/PbqDgBHm/9pJ7Aq3TlOvjbr2KUsVA==", + "license": "MIT", + "dependencies": { + "@loaders.gl/geoarrow": "4.4.4", + "@loaders.gl/loader-utils": "4.4.4", + "@loaders.gl/schema": "4.4.4", + "@loaders.gl/schema-utils": "4.4.4", + "@mapbox/vector-tile": "^1.3.1", + "@math.gl/polygon": "^4.1.0", + "pbf": "^3.2.1" + }, + "peerDependencies": { + "@loaders.gl/core": "~4.4.0" + } + }, + "node_modules/@loaders.gl/gltf": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/gltf/-/gltf-4.4.4.tgz", + "integrity": "sha512-EVslafU3LWjOkuoHH5Oh+sF4WuOdG/ZQf/WwTTzlZsoFhkOFG5Hj1Wmk4kLOY8bYZj7RhspDuDW/GIW45zaEnQ==", + "license": "MIT", + "dependencies": { + "@loaders.gl/draco": "4.4.4", + "@loaders.gl/images": "4.4.4", + "@loaders.gl/loader-utils": "4.4.4", + "@loaders.gl/schema": "4.4.4", + "@loaders.gl/textures": "4.4.4", + "@math.gl/core": "^4.1.0" + }, + "peerDependencies": { + "@loaders.gl/core": "~4.4.0" + } + }, + "node_modules/@loaders.gl/images": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/images/-/images-4.4.4.tgz", + "integrity": "sha512-pqCfWRR7yEHCnAaWDu1aoD7NILts6mvY+B47lQXJ00Ffm4/cfx35dTZ4GUung/iqdVAbxC7fpGKwTPn3T06x4w==", + "license": "MIT", + "dependencies": { + "@loaders.gl/loader-utils": "4.4.4" + }, + "peerDependencies": { + "@loaders.gl/core": "~4.4.0" + } + }, + "node_modules/@loaders.gl/loader-utils": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/loader-utils/-/loader-utils-4.4.4.tgz", + "integrity": "sha512-+0IizJUB6GvPQjl2xC2V1vpcMPcidZykWy27LFnIIqG1lesxfALnFmRoq9MjbOj2TPcUo7VSBxl8z55qRlX0rQ==", + "license": "MIT", + "dependencies": { + "@loaders.gl/schema": "4.4.4", + "@loaders.gl/worker-utils": "4.4.4", + "@probe.gl/log": "^4.1.1", + "@probe.gl/stats": "^4.1.1" + } + }, + "node_modules/@loaders.gl/math": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/math/-/math-4.4.4.tgz", + "integrity": "sha512-cKouXGDpQZQRXJIqt/SelsC5XbslhhSSlFof9fXnS17uX9KROq0Wtt70k0kr+PIG8kFgTerrdcGNM3DIqpQ/hQ==", + "license": "MIT", + "dependencies": { + "@math.gl/core": "^4.1.0" + }, + "peerDependencies": { + "@loaders.gl/core": "~4.4.0" + } + }, + "node_modules/@loaders.gl/mvt": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/mvt/-/mvt-4.4.4.tgz", + "integrity": "sha512-njxMrrkRnBEWFWACzh1Npmk74QZ/hXxonoJEVQ5jTRIA41Ny4KoqxtCa3w3qphiCDGTbGi0cDeDKTp+9d3s0LQ==", + "license": "MIT", + "dependencies": { + "@loaders.gl/gis": "4.4.4", + "@loaders.gl/images": "4.4.4", + "@loaders.gl/loader-utils": "4.4.4", + "@loaders.gl/schema": "4.4.4", + "@loaders.gl/schema-utils": "4.4.4", + "@math.gl/polygon": "^4.1.0", + "@probe.gl/stats": "^4.1.1", + "pbf": "^3.2.1" + }, + "peerDependencies": { + "@loaders.gl/core": "~4.4.0" + } + }, + "node_modules/@loaders.gl/schema": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/schema/-/schema-4.4.4.tgz", + "integrity": "sha512-3spgUoN505V2OprDDzww5nhWfn8Htupcfdr+/lxswAhCbGjwM3RWzA3Zif3dW6ZxOCeSIRJ4GNON78Eiede2dw==", + "license": "MIT", + "dependencies": { + "@types/geojson": "^7946.0.7", + "apache-arrow": ">= 17.0.0" + } + }, + "node_modules/@loaders.gl/schema-utils": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/schema-utils/-/schema-utils-4.4.4.tgz", + "integrity": "sha512-/ykSrQlfn7Lx+kmQNuGzTaywLVy2RoaAZ5YiEXkP9UJ+4lYY2wP5KgUlQGwODDdzNPLFmYzd+FXeE9GEmlXIUg==", + "license": "MIT", + "dependencies": { + "@loaders.gl/schema": "4.4.4", + "@math.gl/types": "^4.1.0", + "@types/geojson": "^7946.0.7", + "apache-arrow": ">= 17.0.0" + }, + "peerDependencies": { + "@loaders.gl/core": "~4.4.0" + } + }, + "node_modules/@loaders.gl/terrain": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/terrain/-/terrain-4.4.4.tgz", + "integrity": "sha512-eGOwBzmI1SVsG5zDvpi5Iw6Np3JdklOOeU/AGAC/vRU/tMqIvYk9qT77mZkR71bKQXkmBEMPRzdv6je9HEFzjg==", + "license": "MIT", + "dependencies": { + "@loaders.gl/images": "4.4.4", + "@loaders.gl/loader-utils": "4.4.4", + "@loaders.gl/schema": "4.4.4", + "@mapbox/martini": "^0.2.0" + }, + "peerDependencies": { + "@loaders.gl/core": "~4.4.0" + } + }, + "node_modules/@loaders.gl/textures": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/textures/-/textures-4.4.4.tgz", + "integrity": "sha512-O0CMKZpOY0kH/s6oSPfzVkUeGgiJnlPFpPL2KJFo1NBkJihfnzAisfPrU4FEVFYm6XfbclkB5BC+6mI2DXgRLQ==", + "license": "MIT", + "dependencies": { + "@loaders.gl/images": "4.4.4", + "@loaders.gl/loader-utils": "4.4.4", + "@loaders.gl/schema": "4.4.4", + "@loaders.gl/worker-utils": "4.4.4", + "@math.gl/types": "^4.1.0", + "ktx-parse": "^0.7.0", + "texture-compressor": "^1.0.2" + }, + "peerDependencies": { + "@loaders.gl/core": "~4.4.0" + } + }, + "node_modules/@loaders.gl/tiles": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/tiles/-/tiles-4.4.4.tgz", + "integrity": "sha512-2dspQParviZmDNi1naH6xi9SmUHkEp4ADt5CEOGVIx/pFsmKG7cAtVvvpWGZ1c4pGSUvCWPjlv/QcA79p7EMGQ==", + "license": "MIT", + "dependencies": { + "@loaders.gl/loader-utils": "4.4.4", + "@loaders.gl/math": "4.4.4", + "@math.gl/core": "^4.1.0", + "@math.gl/culling": "^4.1.0", + "@math.gl/geospatial": "^4.1.0", + "@math.gl/web-mercator": "^4.1.0", + "@probe.gl/stats": "^4.1.1" + }, + "peerDependencies": { + "@loaders.gl/core": "~4.4.0" + } + }, + "node_modules/@loaders.gl/wms": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/wms/-/wms-4.4.4.tgz", + "integrity": "sha512-D42D69IUAXRkgDfIzobKmiaQA9ktsAzKKBE2bWSwfze+5upsTUycmBMV95qeqFl5AEXtxU92EJByA4MCvDnkpw==", + "license": "MIT", + "dependencies": { + "@loaders.gl/images": "4.4.4", + "@loaders.gl/loader-utils": "4.4.4", + "@loaders.gl/schema": "4.4.4", + "@loaders.gl/xml": "4.4.4", + "@turf/rewind": "^5.1.5", + "deep-strict-equal": "^0.2.0" + }, + "peerDependencies": { + "@loaders.gl/core": "~4.4.0" + } + }, + "node_modules/@loaders.gl/worker-utils": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/worker-utils/-/worker-utils-4.4.4.tgz", + "integrity": "sha512-UNQCwK7tHd7Qh0K7e6Flvac8qd14SoAgaN8SOjggd5ngYqwytOQiAo5uEuhRN+r7+EqdwU+qwcpRtjYByT+RqA==", + "license": "MIT", + "peerDependencies": { + "@loaders.gl/core": "~4.4.0" + } + }, + "node_modules/@loaders.gl/xml": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/xml/-/xml-4.4.4.tgz", + "integrity": "sha512-/EtNcjISCeTO3VBwkkz5j68fmaBBaXfaTakP+urg0GdBBGEflZwbmBNa9eJP6ObZNR1EQRHyylzjcjIFaOCJmA==", + "license": "MIT", + "dependencies": { + "@loaders.gl/loader-utils": "4.4.4", + "@loaders.gl/schema": "4.4.4", + "fast-xml-parser": "^5.3.6" + }, + "peerDependencies": { + "@loaders.gl/core": "~4.4.0" + } + }, + "node_modules/@loaders.gl/zip": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@loaders.gl/zip/-/zip-4.4.4.tgz", + "integrity": "sha512-XzHx8/sfvn8tgfuHOXV8xyyuOMiA0STP5KUvkXD6ntiE37AtShNyseSvjLn2GF3vpuTUuWJ8wgKU6O3+H89s8A==", + "license": "MIT", + "dependencies": { + "@loaders.gl/compression": "4.4.4", + "@loaders.gl/crypto": "4.4.4", + "@loaders.gl/loader-utils": "4.4.4", + "jszip": "^3.1.5", + "md5": "^2.3.0" + }, + "peerDependencies": { + "@loaders.gl/core": "~4.4.0" + } + }, + "node_modules/@luma.gl/core": { + "version": "9.3.6", + "resolved": "https://registry.npmjs.org/@luma.gl/core/-/core-9.3.6.tgz", + "integrity": "sha512-eqHnCPh2xHYkdd9rEkiIIkGMixNiJkq1ROrnTob6npvmZNVHXL0ubIw5KueL7rqW0+J1tm+p2+TXZaCmn0sP7Q==", + "license": "MIT", + "dependencies": { + "@math.gl/types": "^4.1.0", + "@probe.gl/env": "^4.1.1", + "@probe.gl/log": "^4.1.1", + "@probe.gl/stats": "^4.1.1", + "@types/offscreencanvas": "^2019.7.3" + } + }, + "node_modules/@luma.gl/engine": { + "version": "9.3.6", + "resolved": "https://registry.npmjs.org/@luma.gl/engine/-/engine-9.3.6.tgz", + "integrity": "sha512-NYTdhn2NaH/MjN8qXCl2ukrT1Ac9iTKu1mgN0wz11XRd1QYnlMNBXswRROD7IZ2PUsBWdmYHc9qE8wKBQqMuIA==", + "license": "MIT", + "dependencies": { + "@math.gl/core": "^4.1.0", + "@math.gl/types": "^4.1.0", + "@probe.gl/log": "^4.1.1", + "@probe.gl/stats": "^4.1.1" + }, + "peerDependencies": { + "@luma.gl/core": "~9.3.0", + "@luma.gl/shadertools": "~9.3.0" + } + }, + "node_modules/@luma.gl/gltf": { + "version": "9.3.6", + "resolved": "https://registry.npmjs.org/@luma.gl/gltf/-/gltf-9.3.6.tgz", + "integrity": "sha512-9R27aYwvu6KBJzd9Kxs4cqIJpgJsI1AyL6Pn6+5CrmGbaCqMtgWi7Sf9Wo+bqv0PRxwzH0j1oAv7qRk0hGbstw==", + "license": "MIT", + "dependencies": { + "@loaders.gl/core": "~4.4.0", + "@loaders.gl/gltf": "~4.4.0", + "@loaders.gl/textures": "~4.4.0", + "@math.gl/core": "^4.1.0" + }, + "peerDependencies": { + "@luma.gl/core": "~9.3.0", + "@luma.gl/engine": "~9.3.0", + "@luma.gl/shadertools": "~9.3.0" + } + }, + "node_modules/@luma.gl/shadertools": { + "version": "9.3.6", + "resolved": "https://registry.npmjs.org/@luma.gl/shadertools/-/shadertools-9.3.6.tgz", + "integrity": "sha512-dDuD8lCOkAE1L7hJGZEyZAi7upR1HG3wEEIQ1gCLaTTbJWC7tNtnVz853lqUA+VXgjgS9NiQFFRcosiIZmW4Vg==", + "license": "MIT", + "dependencies": { + "@math.gl/core": "^4.1.0", + "@math.gl/types": "^4.1.0" + }, + "peerDependencies": { + "@luma.gl/core": "~9.3.0" + } + }, + "node_modules/@luma.gl/webgl": { + "version": "9.3.6", + "resolved": "https://registry.npmjs.org/@luma.gl/webgl/-/webgl-9.3.6.tgz", + "integrity": "sha512-tGm7FGWPmJKxGZMvkRPRi219x3tKmBxR7Uke09nTp2ujNak/O5V9xPIQ9lUBGTxqAb8U2yjKkXG8j+f/4b2+sw==", + "license": "MIT", + "dependencies": { + "@math.gl/types": "^4.1.0", + "@probe.gl/env": "^4.1.1" + }, + "peerDependencies": { + "@luma.gl/core": "~9.3.0" + } + }, + "node_modules/@mapbox/martini": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@mapbox/martini/-/martini-0.2.0.tgz", + "integrity": "sha512-7hFhtkb0KTLEls+TRw/rWayq5EeHtTaErgm/NskVoXmtgAQu/9D299aeyj6mzAR/6XUnYRp2lU+4IcrYRFjVsQ==", + "license": "ISC" + }, + "node_modules/@mapbox/point-geometry": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@mapbox/point-geometry/-/point-geometry-0.1.0.tgz", + "integrity": "sha512-6j56HdLTwWGO0fJPlrZtdU/B13q8Uwmo18Ck2GnGgN9PCFyKTZ3UbXeEdRFh18i9XQ92eH2VdtpJHpBD3aripQ==", + "license": "ISC" + }, + "node_modules/@mapbox/tiny-sdf": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@mapbox/tiny-sdf/-/tiny-sdf-2.2.0.tgz", + "integrity": "sha512-LVL4wgI9YAum5V+LNVQO6QgFBPw7/MIIY4XJPNsPDMrjEwcE+JfKk1LuIl8GnF197ejVdC9QdPaxrx5gfgdGXg==", + "license": "BSD-2-Clause" + }, + "node_modules/@mapbox/vector-tile": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@mapbox/vector-tile/-/vector-tile-1.3.1.tgz", + "integrity": "sha512-MCEddb8u44/xfQ3oD+Srl/tNcQoqTw3goGk2oLsrFxOTc3dUp+kAnby3PvAeeBYSMSjSPD1nd1AJA6W49WnoUw==", + "license": "BSD-3-Clause", + "dependencies": { + "@mapbox/point-geometry": "~0.1.0" + } + }, + "node_modules/@math.gl/core": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@math.gl/core/-/core-4.1.0.tgz", + "integrity": "sha512-FrdHBCVG3QdrworwrUSzXIaK+/9OCRLscxI2OUy6sLOHyHgBMyfnEGs99/m3KNvs+95BsnQLWklVfpKfQzfwKA==", + "license": "MIT", + "dependencies": { + "@math.gl/types": "4.1.0" + } + }, + "node_modules/@math.gl/culling": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@math.gl/culling/-/culling-4.1.0.tgz", + "integrity": "sha512-jFmjFEACnP9kVl8qhZxFNhCyd47qPfSVmSvvjR0/dIL6R9oD5zhR1ub2gN16eKDO/UM7JF9OHKU3EBIfeR7gtg==", + "license": "MIT", + "dependencies": { + "@math.gl/core": "4.1.0", + "@math.gl/types": "4.1.0" + } + }, + "node_modules/@math.gl/geospatial": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@math.gl/geospatial/-/geospatial-4.1.0.tgz", + "integrity": "sha512-BzsUhpVvnmleyYF6qdqJIip6FtIzJmnWuPTGhlBuPzh7VBHLonCFSPtQpbkRuoyAlbSyaGXcVt6p6lm9eK2vtg==", + "license": "MIT", + "dependencies": { + "@math.gl/core": "4.1.0", + "@math.gl/types": "4.1.0" + } + }, + "node_modules/@math.gl/polygon": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@math.gl/polygon/-/polygon-4.1.0.tgz", + "integrity": "sha512-YA/9PzaCRHbIP5/0E9uTYrqe+jsYTQoqoDWhf6/b0Ixz8bPZBaGDEafLg3z7ffBomZLacUty9U3TlPjqMtzPjA==", + "license": "MIT", + "dependencies": { + "@math.gl/core": "4.1.0" + } + }, + "node_modules/@math.gl/sun": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@math.gl/sun/-/sun-4.1.0.tgz", + "integrity": "sha512-i3q6OCBLSZ5wgZVhXg+X7gsjY/TUtuFW/2KBiq/U1ypLso3S4sEykoU/MGjxUv1xiiGtr+v8TeMbO1OBIh/HmA==", + "license": "MIT" + }, + "node_modules/@math.gl/types": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@math.gl/types/-/types-4.1.0.tgz", + "integrity": "sha512-clYZdHcmRvMzVK5fjeDkQlHUzXQSNdZ7s4xOqC3nJPgz4C/TZkUecTo9YS4PruZqtDda/ag4erndP0MIn40dGA==", + "license": "MIT" + }, + "node_modules/@math.gl/web-mercator": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@math.gl/web-mercator/-/web-mercator-4.1.0.tgz", + "integrity": "sha512-HZo3vO5GCMkXJThxRJ5/QYUYRr3XumfT8CzNNCwoJfinxy5NtKUd7dusNTXn7yJ40UoB8FMIwkVwNlqaiRZZAw==", + "license": "MIT", + "dependencies": { + "@math.gl/core": "4.1.0" + } + }, + "node_modules/@napi-rs/lzma-linux-x64-gnu": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-x64-gnu/-/lzma-linux-x64-gnu-1.5.1.tgz", + "integrity": "sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.20 || ^24.12 || >=25" + } + }, + "node_modules/@nodable/entities": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", + "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, + "node_modules/@open-wc/dedupe-mixin": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@open-wc/dedupe-mixin/-/dedupe-mixin-1.4.0.tgz", + "integrity": "sha512-Sj7gKl1TLcDbF7B6KUhtvr+1UCxdhMbNY5KxdU5IfMFWqL8oy1ZeAcCANjoB1TL0AJTcPmcCFsCbHf8X2jGDUA==", + "license": "MIT", + "peer": true + }, + "node_modules/@pact-tailor/api": { + "resolved": "packages/api", + "link": true + }, + "node_modules/@pact-tailor/connectors": { + "resolved": "packages/connectors", + "link": true + }, + "node_modules/@pact-tailor/globe": { + "resolved": "packages/globe", + "link": true + }, + "node_modules/@pact-tailor/graph": { + "resolved": "packages/graph", + "link": true + }, + "node_modules/@pact-tailor/intel": { + "resolved": "packages/intel", + "link": true + }, + "node_modules/@pact-tailor/ontology": { + "resolved": "packages/ontology", + "link": true + }, + "node_modules/@pact-tailor/pact-bridge": { + "resolved": "packages/pact-bridge", + "link": true + }, + "node_modules/@pact-tailor/registry": { + "resolved": "packages/registry", + "link": true + }, + "node_modules/@pact-tailor/state": { + "resolved": "packages/state", + "link": true + }, + "node_modules/@pact-tailor/store": { + "resolved": "packages/store", + "link": true + }, + "node_modules/@polymer/polymer": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/@polymer/polymer/-/polymer-3.5.2.tgz", + "integrity": "sha512-fWwImY/UH4bb2534DVSaX+Azs2yKg8slkMBHOyGeU2kKx7Xmxp6Lee0jP8p6B3d7c1gFUPB2Z976dTUtX81pQA==", + "license": "BSD-3-Clause", + "peer": true, + "dependencies": { + "@webcomponents/shadycss": "^1.9.1" + } + }, + "node_modules/@probe.gl/env": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@probe.gl/env/-/env-4.1.1.tgz", + "integrity": "sha512-+68seNDMVsEegRB47pFA/Ws1Fjy8agcFYXxzorKToyPcD6zd+gZ5uhwoLd7TzsSw6Ydns//2KEszWn+EnNHTbA==", + "license": "MIT" + }, + "node_modules/@probe.gl/log": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@probe.gl/log/-/log-4.1.1.tgz", + "integrity": "sha512-kcZs9BT44pL7hS1OkRGKYRXI/SN9KejUlPD+BY40DguRLzdC5tLG/28WGMyfKdn/51GT4a0p+0P8xvDn1Ez+Kg==", + "license": "MIT", + "dependencies": { + "@probe.gl/env": "4.1.1" + } + }, + "node_modules/@probe.gl/stats": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@probe.gl/stats/-/stats-4.1.1.tgz", + "integrity": "sha512-4VpAyMHOqydSvPlEyHwXaE+AkIdR03nX+Qhlxsk2D/IW4OVmDZgIsvJB1cDzyEEtcfKcnaEbfXeiPgejBceT6g==", + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.62.4.tgz", + "integrity": "sha512-RrPokAb7dmbxFoeO3TloqHyOjgye8RkBhSqmp4aJMIex4c9r46ZstPnleDQOq1t46VOVjwIuwNogIqbodV1Vvg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.62.4.tgz", + "integrity": "sha512-JKuJc+pnpks2pjy7L/N3v/cAkZxYlnmuZoD840ldbMI5KDbC4iO9NKwPKYdjYFCMAIIlBzYSFHxIJVYzRo2/8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.62.4.tgz", + "integrity": "sha512-krw5uS2STmvJ02x0uTXHbqQNuz+9eZ1iw+qXk9dmW2gvV4jV7O2hEoOnuhFrpOPiel1mBFtqbxYZZtC46hXLOw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.62.4.tgz", + "integrity": "sha512-wsTxtgApb4PrOsNJIm0FZ1h3WvCC+k9uxLJ4ad75hgoS4NiRes2SoJFlDAyMwiUY8IssDqGcHbXuN0sx1tfF1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.62.4.tgz", + "integrity": "sha512-GUOnQlyZe3yAXhWOtOMsn5Qkrv5E5mZXa0thbARWi5Ei2szlVXJFQhddZ4HbAzh8q92w5twp+CQvs/eFanz9YQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.62.4.tgz", + "integrity": "sha512-/Y7f3QuxjzPKsjA/rfEDa3+0vXqyjmJ50Ln8dPpCmWkKTrUoWHG1cWhTqaAMLob2m2nESWuC7yGrREz019Ztqg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.62.4.tgz", + "integrity": "sha512-81wiiX3v7aqy+T+bT61TJ78yJjRquqFFTTbAPt08imfQQzkPIW8t6aJbkTagtCCrXMNc9D66+geqlK7ydLPNqA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.62.4.tgz", + "integrity": "sha512-9kmDIvNZqdoHOBZgNtpTBeLWYO/LVipM3H/j62P8848/l/VPEQL6N3uxU9pvP1oZAsXyC2MEnFP3ovRjo7WYNQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.62.4.tgz", + "integrity": "sha512-CcnXHWnXg69g+DX5VWL3FHts3qMRN2uVEHX+BZvGLdd07/gXkn3ePjYtO1LDJvxkGKVHMclKBRa1QUTH+6toYQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.62.4.tgz", + "integrity": "sha512-iFOibiHnTRuhrWLlRsOQFdZJJIa7S8OwkneJr4ocALP16u5yk6lWLINFwhHaEqBFMsKDUZofLkGos7+CPzGB3g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.62.4.tgz", + "integrity": "sha512-XnWYMI7euHlb5a871xPja+Gm7DRCFU+FGRrtS2sMq9N8FvqtpagUy6gD4YOemC5MRk9xbh8+jYMEJbigFQwsgA==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.62.4.tgz", + "integrity": "sha512-qGDAlO0U8xedCcsdRm9oaoQY8DAx/QT7uIxJWhCdx0ceIWX783UC9QSYkdpzAe29wNiVfp24+bZdQmn49o45SQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.62.4.tgz", + "integrity": "sha512-ru4H6ezD7ysA5EiEK6qkkaEb4modH8CTej6kUy/gQi20u3kB3G7Zn8snXXkeJSCOFKG/rbPPtM/+9Wgas1961w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.62.4.tgz", + "integrity": "sha512-2W4MO5WQVJnbJaZdvDb9rhBDuFU1nKIepPFpJUBsTh2k1YY2g+ODViaWuyOAjQ5cOP7NvrvLzt3wvHOoiAvc7w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.62.4.tgz", + "integrity": "sha512-+fxjfuoAmVMCYV5QyjoIpu0cp5DOiOTeqYFk1AVaxGr+/ravWLX89XfQmptsoWcaVy/TGf2hexzbUOrCQIL1CQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.62.4.tgz", + "integrity": "sha512-jTn8JfHGL4djjFxPuM06LmNUJDsst2jeVlsd9OmIH6zc5sC9K6rIuO4YajXatLUpBmBKl6b35ro1QZocLi+tcA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.62.4.tgz", + "integrity": "sha512-oCJCJL4pXsoDcP2QZ+JVlPTIRc6266zsIaeJJsWImmF7HO0W8nb6HuSgZlMWxJwaPf8ehbSw8yo0EUw925hKsA==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.62.4.tgz", + "integrity": "sha512-W69hukhZ3KKNRCaMIEzKvcFye42hh0FE1+YoYaf5+Ikacuftoco6yO/xouz0hc5d5W/s3yBro5jRiuEE/Q5vUw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.62.4.tgz", + "integrity": "sha512-qiXbGG2jkjXhzXpsFZSR2Xpb8DN/UaxYsbb/STbuR/6fpaDgRmmaq1B/LmtF2wQFOFOSsK2jdE0RZ3a0zHn4QA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.62.4.tgz", + "integrity": "sha512-nWeM//hxv8mIo6jD7Hu4o48DVmV9pbV6gsKaWU+4NFyqHoPKwrkRiZGLKUhOBk8qNmDmpwFtPKg80Bo/Tn4xiQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.62.4.tgz", + "integrity": "sha512-s62SQ/vgsRSvMwDkOEfTqfgASF0f26ZNaQuTA6Aok5lrikf89yI2W0gFHvZb2Jpgc6N8JnOKZgCK2iciO3CsxQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.62.4.tgz", + "integrity": "sha512-J6wGf8TVGbXJq+HH+ttTvrcfNKPbuZecV6KT1B8I18BC5IURUh5kl4Yl5OEP5eFIUoI5BWxCsyYMhFsDx8kekw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.62.4.tgz", + "integrity": "sha512-zmfrQd/0wu6oJs8Vq8KwY/YtsKSsLtKe/HwAP4Wqy8LhWjeT55fHRAkOhYQ12wI3ayS4Tt12d5CDRD7N96SAYQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.62.4.tgz", + "integrity": "sha512-qPzHqdj9rfUD+w79dtE07zi/kFwKyCJqplp5K5ygeLTp7jLpAoc16OAH39HSmRC9UpozaecsleI8uAdEj6v2yw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.62.4.tgz", + "integrity": "sha512-zD6NdeWEByGE9QF9vCrlJ5YQB4oq9q91kPZS37Jwj5hOkvR1lTBSpsKhKDw4IJtbQ35LsTS1HD9DZYGKIshU1Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@turf/boolean-clockwise": { + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/@turf/boolean-clockwise/-/boolean-clockwise-5.1.5.tgz", + "integrity": "sha512-FqbmEEOJ4rU4/2t7FKx0HUWmjFEVqR+NJrFP7ymGSjja2SQ7Q91nnBihGuT+yuHHl6ElMjQ3ttsB/eTmyCycxA==", + "license": "MIT", + "dependencies": { + "@turf/helpers": "^5.1.5", + "@turf/invariant": "^5.1.5" + } + }, + "node_modules/@turf/clone": { + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/@turf/clone/-/clone-5.1.5.tgz", + "integrity": "sha512-//pITsQ8xUdcQ9pVb4JqXiSqG4dos5Q9N4sYFoWghX21tfOV2dhc5TGqYOhnHrQS7RiKQL1vQ48kIK34gQ5oRg==", + "license": "MIT", + "dependencies": { + "@turf/helpers": "^5.1.5" + } + }, + "node_modules/@turf/helpers": { + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/@turf/helpers/-/helpers-5.1.5.tgz", + "integrity": "sha512-/lF+JR+qNDHZ8bF9d+Cp58nxtZWJ3sqFe6n3u3Vpj+/0cqkjk4nXKYBSY0azm+GIYB5mWKxUXvuP/m0ZnKj1bw==", + "license": "MIT" + }, + "node_modules/@turf/invariant": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@turf/invariant/-/invariant-5.2.0.tgz", + "integrity": "sha512-28RCBGvCYsajVkw2EydpzLdcYyhSA77LovuOvgCJplJWaNVyJYH6BOR3HR9w50MEkPqb/Vc/jdo6I6ermlRtQA==", + "license": "MIT", + "dependencies": { + "@turf/helpers": "^5.1.5" + } + }, + "node_modules/@turf/meta": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@turf/meta/-/meta-5.2.0.tgz", + "integrity": "sha512-ZjQ3Ii62X9FjnK4hhdsbT+64AYRpaI8XMBMcyftEOGSmPMUVnkbvuv3C9geuElAXfQU7Zk1oWGOcrGOD9zr78Q==", + "license": "MIT", + "dependencies": { + "@turf/helpers": "^5.1.5" + } + }, + "node_modules/@turf/rewind": { + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/@turf/rewind/-/rewind-5.1.5.tgz", + "integrity": "sha512-Gdem7JXNu+G4hMllQHXRFRihJl3+pNl7qY+l4qhQFxq+hiU1cQoVFnyoleIqWKIrdK/i2YubaSwc3SCM7N5mMw==", + "license": "MIT", + "dependencies": { + "@turf/boolean-clockwise": "^5.1.5", + "@turf/clone": "^5.1.5", + "@turf/helpers": "^5.1.5", + "@turf/invariant": "^5.1.5", + "@turf/meta": "^5.1.5" + } + }, + "node_modules/@types/brotli": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/brotli/-/brotli-1.3.5.tgz", + "integrity": "sha512-9xoNr+bcxT236/7ZgcWw/6Pb2RRetE13p4bFy1xYSckKwyOiRfmInay8baUWZgH7/284Wl6IPe7+nOI9+OQg/A==", + "license": "MIT", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/crypto-js": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/@types/crypto-js/-/crypto-js-4.2.2.tgz", + "integrity": "sha512-sDOLlVbHhXpAUAL0YHDUUwDZf3iN4Bwi4W6a0W0b+QcAezUbRtH4FVb+9J4h+XFPW7l/gQ9F8qC7P+Ec4k8QVQ==", + "license": "MIT" + }, + "node_modules/@types/d3": { + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/@types/d3/-/d3-7.4.3.tgz", + "integrity": "sha512-lZXZ9ckh5R8uiFVt8ogUNf+pIrK4EsWrx2Np75WvF/eTpJ0FMHNhjXk8CKEx/+gpHbNQyJWehbFaTvqmHWB3ww==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/d3-array": "*", + "@types/d3-axis": "*", + "@types/d3-brush": "*", + "@types/d3-chord": "*", + "@types/d3-color": "*", + "@types/d3-contour": "*", + "@types/d3-delaunay": "*", + "@types/d3-dispatch": "*", + "@types/d3-drag": "*", + "@types/d3-dsv": "*", + "@types/d3-ease": "*", + "@types/d3-fetch": "*", + "@types/d3-force": "*", + "@types/d3-format": "*", + "@types/d3-geo": "*", + "@types/d3-hierarchy": "*", + "@types/d3-interpolate": "*", + "@types/d3-path": "*", + "@types/d3-polygon": "*", + "@types/d3-quadtree": "*", + "@types/d3-random": "*", + "@types/d3-scale": "*", + "@types/d3-scale-chromatic": "*", + "@types/d3-selection": "*", + "@types/d3-shape": "*", + "@types/d3-time": "*", + "@types/d3-time-format": "*", + "@types/d3-timer": "*", + "@types/d3-transition": "*", + "@types/d3-zoom": "*" + } + }, + "node_modules/@types/d3-array": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.2.tgz", + "integrity": "sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw==", + "license": "MIT" + }, + "node_modules/@types/d3-axis": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-axis/-/d3-axis-3.0.6.tgz", + "integrity": "sha512-pYeijfZuBd87T0hGn0FO1vQ/cgLk6E1ALJjfkC0oJ8cbwkZl3TpgS8bVBLZN+2jjGgg38epgxb2zmoGtSfvgMw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-brush": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-brush/-/d3-brush-3.0.6.tgz", + "integrity": "sha512-nH60IZNNxEcrh6L1ZSMNA28rj27ut/2ZmI3r96Zd+1jrZD++zD3LsMIjWlvg4AYrHn/Pqz4CF3veCxGjtbqt7A==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-chord": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-chord/-/d3-chord-3.0.6.tgz", + "integrity": "sha512-LFYWWd8nwfwEmTZG9PfQxd17HbNPksHBiJHaKuY1XeqscXacsS2tyoo6OdRsjf+NQYeB6XrNL3a25E3gH69lcg==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/d3-color": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz", + "integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==", + "license": "MIT" + }, + "node_modules/@types/d3-contour": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-contour/-/d3-contour-3.0.6.tgz", + "integrity": "sha512-BjzLgXGnCWjUSYGfH1cpdo41/hgdWETu4YxpezoztawmqsvCeep+8QGfiY6YbDvfgHz/DkjeIkkZVJavB4a3rg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/d3-array": "*", + "@types/geojson": "*" + } + }, + "node_modules/@types/d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-ZMaSKu4THYCU6sV64Lhg6qjf1orxBthaC161plr5KuPHo3CNm8DTHiLw/5Eq2b6TsNP0W0iJrUOFscY6Q450Hw==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/d3-dispatch": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-dispatch/-/d3-dispatch-3.0.7.tgz", + "integrity": "sha512-5o9OIAdKkhN1QItV2oqaE5KMIiXAvDWBDPrD85e58Qlz1c1kI/J0NcqbEG88CoTwJrYe7ntUCVfeUl2UJKbWgA==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/d3-drag": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-drag/-/d3-drag-3.0.7.tgz", + "integrity": "sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-dsv": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-dsv/-/d3-dsv-3.0.7.tgz", + "integrity": "sha512-n6QBF9/+XASqcKK6waudgL0pf/S5XHPPI8APyMLLUHd8NqouBGLsU8MgtO7NINGtPBtk9Kko/W4ea0oAspwh9g==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/d3-ease": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz", + "integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/d3-fetch": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-fetch/-/d3-fetch-3.0.7.tgz", + "integrity": "sha512-fTAfNmxSb9SOWNB9IoG5c8Hg6R+AzUHDRlsXsDZsNp6sxAEOP0tkP3gKkNSO/qmHPoBFTxNrjDprVHDQDvo5aA==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/d3-dsv": "*" + } + }, + "node_modules/@types/d3-force": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@types/d3-force/-/d3-force-3.0.10.tgz", + "integrity": "sha512-ZYeSaCF3p73RdOKcjj+swRlZfnYpK1EbaDiYICEEp5Q6sUiqFaFQ9qgoshp5CzIyyb/yD09kD9o2zEltCexlgw==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/d3-format": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-format/-/d3-format-3.0.4.tgz", + "integrity": "sha512-fALi2aI6shfg7vM5KiR1wNJnZ7r6UuggVqtDA+xiEdPZQwy/trcQaHnwShLuLdta2rTymCNpxYTiMZX/e09F4g==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/d3-geo": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@types/d3-geo/-/d3-geo-3.1.1.tgz", + "integrity": "sha512-65Emv9fQiQQqphLlRkuQ5ypPsOmWPhtBGCMv61JDPEPMvsx+gzhGf74yw1a78xFKPj6zw4AgQICJoQv0vK9M2w==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/geojson": "*" + } + }, + "node_modules/@types/d3-hierarchy": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@types/d3-hierarchy/-/d3-hierarchy-3.1.1.tgz", + "integrity": "sha512-QwjxA3+YCKH3N1Rs3uSiSy1bdxlLB1uUiENXeJudBoAFvtDuswUxLcanoOaR2JYn1melDTuIXR8VhnVyI3yG/A==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/d3-interpolate": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz", + "integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/d3-color": "*" + } + }, + "node_modules/@types/d3-path": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.1.1.tgz", + "integrity": "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/d3-polygon": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-polygon/-/d3-polygon-3.0.2.tgz", + "integrity": "sha512-ZuWOtMaHCkN9xoeEMr1ubW2nGWsp4nIql+OPQRstu4ypeZ+zk3YKqQT0CXVe/PYqrKpZAi+J9mTs05TKwjXSRA==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/d3-quadtree": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-quadtree/-/d3-quadtree-3.0.6.tgz", + "integrity": "sha512-oUzyO1/Zm6rsxKRHA1vH0NEDG58HrT5icx/azi9MF1TWdtttWl0UIUsjEQBBh+SIkrpd21ZjEv7ptxWys1ncsg==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/d3-random": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-random/-/d3-random-3.0.4.tgz", + "integrity": "sha512-UHYId5WTCx4L4YNel7NU00XUXXgvgpgZOvp10PuvsQENjMDXhh2RyFc0KBjO7B45ne4Ha1yVH7ii0vnzKkuzWA==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/d3-sankey": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/@types/d3-sankey/-/d3-sankey-0.11.2.tgz", + "integrity": "sha512-U6SrTWUERSlOhnpSrgvMX64WblX1AxX6nEjI2t3mLK2USpQrnbwYYK+AS9SwiE7wgYmOsSSKoSdr8aoKBH0HgQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/d3-shape": "^1" + } + }, + "node_modules/@types/d3-sankey/node_modules/@types/d3-path": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-1.0.11.tgz", + "integrity": "sha512-4pQMp8ldf7UaB/gR8Fvvy69psNHkTpD/pVw3vmEi8iZAB9EPMBruB1JvHO4BIq9QkUUd2lV1F5YXpMNj7JPBpw==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/d3-sankey/node_modules/@types/d3-shape": { + "version": "1.3.12", + "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-1.3.12.tgz", + "integrity": "sha512-8oMzcd4+poSLGgV0R1Q1rOlx/xdmozS4Xab7np0eamFFUYq71AU9pOCJEFnkXW2aI/oXdVYJzw6pssbSut7Z9Q==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/d3-path": "^1" + } + }, + "node_modules/@types/d3-scale": { + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-3.3.5.tgz", + "integrity": "sha512-YOpKj0kIEusRf7ofeJcSZQsvKbnTwpe1DUF+P2qsotqG53kEsjm7EzzliqQxMkAWdkZcHrg5rRhB4JiDOQPX+A==", + "license": "MIT", + "dependencies": { + "@types/d3-time": "^2" + } + }, + "node_modules/@types/d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-iWMJgwkK7yTRmWqRB5plb1kadXyQ5Sj8V/zYlFGMUBbIPKQScw+Dku9cAAMgJG+z5GYDoMjWGLVOvjghDEFnKQ==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/d3-scale/node_modules/@types/d3-time": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-2.1.4.tgz", + "integrity": "sha512-BTfLsxTeo7yFxI/haOOf1ZwJ6xKgQLT9dCp+EcmQv87Gox6X+oKl4mLKfO6fnWm3P22+A6DknMNEZany8ql2Rw==", + "license": "MIT" + }, + "node_modules/@types/d3-selection": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-3.0.11.tgz", + "integrity": "sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/d3-shape": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.8.tgz", + "integrity": "sha512-lae0iWfcDeR7qt7rA88BNiqdvPS5pFVPpo5OfjElwNaT2yyekbM0C9vK+yqBqEmHr6lDkRnYNoTBYlAgJa7a4w==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/d3-path": "*" + } + }, + "node_modules/@types/d3-time": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.4.tgz", + "integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/d3-time-format": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-time-format/-/d3-time-format-4.0.3.tgz", + "integrity": "sha512-5xg9rC+wWL8kdDj153qZcsJ0FWiFt0J5RB6LYUNZjwSnesfblqrI/bJ1wBdJ8OQfncgbJG5+2F+qfqnqyzYxyg==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/d3-timer": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz", + "integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/d3-transition": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/@types/d3-transition/-/d3-transition-3.0.9.tgz", + "integrity": "sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-zoom": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@types/d3-zoom/-/d3-zoom-3.0.8.tgz", + "integrity": "sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/d3-interpolate": "*", + "@types/d3-selection": "*" + } + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/geojson": { + "version": "7946.0.16", + "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", + "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", + "license": "MIT" + }, + "node_modules/@types/google.maps": { + "version": "3.65.5", + "resolved": "https://registry.npmjs.org/@types/google.maps/-/google.maps-3.65.5.tgz", + "integrity": "sha512-oR9Hq/WaKGzIguZEwoCuyQ0ofDhCbefmS9RrjLUZ5scMziwTE6xA90drJYt2ZsyUCYmCqnxQcDpfGqeRE/Zgog==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.1.tgz", + "integrity": "sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q==", + "devOptional": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/offscreencanvas": { + "version": "2019.7.3", + "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.7.3.tgz", + "integrity": "sha512-ieXiYmgSRXUDeOntE1InxjWyvEelZGP63M+cGuquuRLuIKKT1osnkXjxev9B7d1nXSug5vpunx+gNlbVxMlC9A==", + "license": "MIT" + }, + "node_modules/@types/pako": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@types/pako/-/pako-1.0.7.tgz", + "integrity": "sha512-YBtzT2ztNF6R/9+UXj2wTGFnC9NklAnASt3sC0h2m1bbH7G6FyBIkt4AN8ThZpNfxUo1b2iMVO0UawiJymEt8A==", + "license": "MIT" + }, + "node_modules/@types/polylabel": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@types/polylabel/-/polylabel-1.1.3.tgz", + "integrity": "sha512-9Zw2KoDpi+T4PZz2G6pO2xArE0m/GSMTW1MIxF2s8ZY8x9XDO6fv9um0ydRGvcbkFLlaq8yNK6eZxnmMZtDgWQ==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/sortablejs": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/@types/sortablejs/-/sortablejs-1.15.9.tgz", + "integrity": "sha512-7HP+rZGE2p886PKV9c9OJzLBI6BBJu1O7lJGYnPyG3fS4/duUCcngkNCjsLwIMV+WMqANe3tt4irrXHSIe68OQ==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/svg-arc-to-cubic-bezier": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/@types/svg-arc-to-cubic-bezier/-/svg-arc-to-cubic-bezier-3.2.3.tgz", + "integrity": "sha512-UNOnbTtl0nVTm8hwKaz5R5VZRvSulFMGojO5+Q7yucKxBoCaTtS4ibSQVRHo5VW5AaRo145U8p1Vfg5KrYe9Bg==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/trusted-types": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", + "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@vaadin/a11y-base": { + "version": "24.9.17", + "resolved": "https://registry.npmjs.org/@vaadin/a11y-base/-/a11y-base-24.9.17.tgz", + "integrity": "sha512-y0uxPHx89aobXvL0q9i81XNJ9oUrINiKmwjXZh9vWygN9Tj02nsBJWad6Q0N1nunEN2M6EvrT4c3vmE1TnI6lg==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/component-base": "~24.9.17", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/checkbox": { + "version": "24.9.17", + "resolved": "https://registry.npmjs.org/@vaadin/checkbox/-/checkbox-24.9.17.tgz", + "integrity": "sha512-iSsjNt4heOatYyVqBKCPooSi3qwauMSPZOdhHKX3bpOvnA2vJRAthtsbIWRyWIxk5i3vAFeh48MGfCvGjCyiHQ==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/a11y-base": "~24.9.17", + "@vaadin/component-base": "~24.9.17", + "@vaadin/field-base": "~24.9.17", + "@vaadin/vaadin-lumo-styles": "~24.9.17", + "@vaadin/vaadin-material-styles": "~24.9.17", + "@vaadin/vaadin-themable-mixin": "~24.9.17", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/component-base": { + "version": "24.9.17", + "resolved": "https://registry.npmjs.org/@vaadin/component-base/-/component-base-24.9.17.tgz", + "integrity": "sha512-nmE1G9bjpaHiEvrUhOaPk0SEHq3OY8lVsimYnqLHwZ9g1DlNzN7YoCZvBy0A4dC8+X9+RKWo7+AK3mbXqawr4w==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/vaadin-development-mode-detector": "^2.0.0", + "@vaadin/vaadin-usage-statistics": "^2.1.0", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/field-base": { + "version": "24.9.17", + "resolved": "https://registry.npmjs.org/@vaadin/field-base/-/field-base-24.9.17.tgz", + "integrity": "sha512-aVda8LDkAzQ7AbFLbKMTvfMlCgrqkC7wPsqjXIASkWV2tIhWQqbnKabGlpf/yopuIUx8l+q90um0Yga5oQ9kqw==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/a11y-base": "~24.9.17", + "@vaadin/component-base": "~24.9.17", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/grid": { + "version": "24.9.17", + "resolved": "https://registry.npmjs.org/@vaadin/grid/-/grid-24.9.17.tgz", + "integrity": "sha512-vZXy9jaqtnugZyZ3XdL43i1E53dVktMEOcg7ZA4IW/KoIeoiy5ioJp+X4JaPCfElY3+rAr0EWIGOcWHrF/dWXg==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/a11y-base": "~24.9.17", + "@vaadin/checkbox": "~24.9.17", + "@vaadin/component-base": "~24.9.17", + "@vaadin/lit-renderer": "~24.9.17", + "@vaadin/text-field": "~24.9.17", + "@vaadin/vaadin-lumo-styles": "~24.9.17", + "@vaadin/vaadin-material-styles": "~24.9.17", + "@vaadin/vaadin-themable-mixin": "~24.9.17", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/icon": { + "version": "24.9.17", + "resolved": "https://registry.npmjs.org/@vaadin/icon/-/icon-24.9.17.tgz", + "integrity": "sha512-yxFBv6SdCGehbhYz5OAbiVeA83dW8eL2oxdeJu7VbyTztxCdnGPioSQN63K9IGVY0dmLlmVNnpA8Li18UgEAdQ==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/component-base": "~24.9.17", + "@vaadin/vaadin-lumo-styles": "~24.9.17", + "@vaadin/vaadin-themable-mixin": "~24.9.17", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/input-container": { + "version": "24.9.17", + "resolved": "https://registry.npmjs.org/@vaadin/input-container/-/input-container-24.9.17.tgz", + "integrity": "sha512-h4zfY4HmNQ9b61e0F/BMCMPkkNcDN2vyVjH64DFQcYwhT42Eay+lGfRo675wj+O1sqvouQ4KjPZS8MsFeVlGHA==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@polymer/polymer": "^3.0.0", + "@vaadin/component-base": "~24.9.17", + "@vaadin/vaadin-lumo-styles": "~24.9.17", + "@vaadin/vaadin-material-styles": "~24.9.17", + "@vaadin/vaadin-themable-mixin": "~24.9.17", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/lit-renderer": { + "version": "24.9.17", + "resolved": "https://registry.npmjs.org/@vaadin/lit-renderer/-/lit-renderer-24.9.17.tgz", + "integrity": "sha512-la43FB3cF5zVtphr8rQas/6AUq8Q/SqR4sYp8jrnXWPGIc8nBqWTMPcIAsJkm6f2SacWc69ymr6GCV0nZQ1QWA==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/text-field": { + "version": "24.9.17", + "resolved": "https://registry.npmjs.org/@vaadin/text-field/-/text-field-24.9.17.tgz", + "integrity": "sha512-WM1luEbUOYZ94EPFhabc37HtMJmCymABw7QuVSnIMa0txZ0UIX77Nr/fJclwB9JUwMTOEf9Sx5UftqqftYg7OA==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/a11y-base": "~24.9.17", + "@vaadin/component-base": "~24.9.17", + "@vaadin/field-base": "~24.9.17", + "@vaadin/input-container": "~24.9.17", + "@vaadin/vaadin-lumo-styles": "~24.9.17", + "@vaadin/vaadin-material-styles": "~24.9.17", + "@vaadin/vaadin-themable-mixin": "~24.9.17", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/vaadin-development-mode-detector": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@vaadin/vaadin-development-mode-detector/-/vaadin-development-mode-detector-2.0.7.tgz", + "integrity": "sha512-9FhVhr0ynSR3X2ao+vaIEttcNU5XfzCbxtmYOV8uIRnUCtNgbvMOIcyGBvntsX9I5kvIP2dV3cFAOG9SILJzEA==", + "license": "Apache-2.0", + "peer": true + }, + "node_modules/@vaadin/vaadin-lumo-styles": { + "version": "24.9.17", + "resolved": "https://registry.npmjs.org/@vaadin/vaadin-lumo-styles/-/vaadin-lumo-styles-24.9.17.tgz", + "integrity": "sha512-5Fjhku/jkopiyBTDNukJMV4j3WwzBN+YUeNK88zxBqr+Qd/61tIzUz5mvH6FLjjKE003FYC9/SlIFfWRzPzIrA==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@polymer/polymer": "^3.0.0", + "@vaadin/component-base": "~24.9.17", + "@vaadin/icon": "~24.9.17", + "@vaadin/vaadin-themable-mixin": "~24.9.17" + } + }, + "node_modules/@vaadin/vaadin-material-styles": { + "version": "24.9.17", + "resolved": "https://registry.npmjs.org/@vaadin/vaadin-material-styles/-/vaadin-material-styles-24.9.17.tgz", + "integrity": "sha512-FCwaV9j7Ox2AR0hpvcZgClMzhZicQku5xIv0TBm99Be2FVQpfo/KAowkoAYQoDpmrXNzpByamG3qF4hvDW451w==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@polymer/polymer": "^3.0.0", + "@vaadin/component-base": "~24.9.17", + "@vaadin/vaadin-themable-mixin": "~24.9.17" + } + }, + "node_modules/@vaadin/vaadin-themable-mixin": { + "version": "24.9.17", + "resolved": "https://registry.npmjs.org/@vaadin/vaadin-themable-mixin/-/vaadin-themable-mixin-24.9.17.tgz", + "integrity": "sha512-3DpdPhhxVjpb3+91YhrfTcmRHn3XM2gMdXWmtSUwLZnUj2zyprJsAL5TK3/MSqVFuPaOwaUWqZXWsk809Ohd7w==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/vaadin-usage-statistics": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@vaadin/vaadin-usage-statistics/-/vaadin-usage-statistics-2.1.3.tgz", + "integrity": "sha512-8r4TNknD7OJQADe3VygeofFR7UNAXZ2/jjBFP5dgI8+2uMfnuGYgbuHivasKr9WSQ64sPej6m8rDoM1uSllXjQ==", + "hasInstallScript": true, + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@vaadin/vaadin-development-mode-detector": "^2.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } + }, + "node_modules/@webcomponents/shadycss": { + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@webcomponents/shadycss/-/shadycss-1.11.2.tgz", + "integrity": "sha512-vRq+GniJAYSBmTRnhCYPAPq6THYqovJ/gzGThWbgEZUQaBccndGTi1hdiUP15HzEco0I6t4RCtXyX0rsSmwgPw==", + "license": "BSD-3-Clause", + "peer": true + }, + "node_modules/@zip.js/zip.js": { + "version": "2.8.41", + "resolved": "https://registry.npmjs.org/@zip.js/zip.js/-/zip.js-2.8.41.tgz", + "integrity": "sha512-w/dbE6JfWIfuLCCKwQEnF6T5MeVc8gpi28lW0Y8bHW7Sbdnb1vq9weP894SvbJzMwOKzpePmFyvGU9v5HYDg9g==", + "license": "BSD-3-Clause", + "peer": true, + "engines": { + "bun": ">=0.7.0", + "deno": ">=1.0.0", + "node": ">=18.0.0" + } + }, + "node_modules/a5-js": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/a5-js/-/a5-js-0.7.3.tgz", + "integrity": "sha512-3aoMwHmNkyuMDHS4q6GRRInpOawamen2pokIbc0MQmR9cqG0Y9+B0bZpzswwetjrSG2ckbYtShH+nKru6+3O5Q==", + "license": "Apache-2.0", + "dependencies": { + "gl-matrix": "^3.4.3" + } + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, + "node_modules/apache-arrow": { + "version": "21.2.0", + "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-21.2.0.tgz", + "integrity": "sha512-Hxe6Agq26gQOM954qpzYSllJBPJl+e16U5CkfuMUhLrNba+5nKkttIVlflaovN6oaTratqMGAO8H5u/aNhmHWQ==", + "license": "Apache-2.0", + "dependencies": { + "@types/node": "^25.2.0", + "flatbuffers": "^25.1.24", + "json-with-bigint": "^3.5.3", + "tslib": "^2.6.2" + }, + "bin": { + "arrow2csv": "bin/arrow2csv.js" + } + }, + "node_modules/apache-arrow/node_modules/@types/node": { + "version": "25.9.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.5.tgz", + "integrity": "sha512-OScDchr2fwuUmWdf4kZ9h7PcJiYDVInhJizG/biAq3cAvqwYktuy/TYGGdZNMtNTFUP7rnb0NU4TUdm82kt4Rg==", + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "node_modules/apache-arrow/node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "license": "MIT" + }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/brotli": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/brotli/-/brotli-1.3.3.tgz", + "integrity": "sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.1.2" + } + }, + "node_modules/browserify-zlib": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.2.0.tgz", + "integrity": "sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA==", + "license": "MIT", + "peer": true, + "dependencies": { + "pako": "~1.0.5" + } + }, + "node_modules/buf-compare": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buf-compare/-/buf-compare-1.0.1.tgz", + "integrity": "sha512-Bvx4xH00qweepGc43xFvMs5BKASXTbHaHm6+kDYIK9p/4iFwjATQkmPKHQSgJZzKbAymhztRbXUf1Nqhzl73/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/call-bind": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.9.tgz", + "integrity": "sha512-a/hy+pNsFUTR+Iz8TCJvXudKVLAnz/DyeSUo10I5yvFDQJBFU2s9uqQpoSrJlroHUKoKqzg+epxyP9lqFdzfBQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "get-intrinsic": "^1.3.0", + "set-function-length": "^1.2.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "peer": true, + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/cartocolor": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/cartocolor/-/cartocolor-5.0.2.tgz", + "integrity": "sha512-Ihb/wU5V6BVbHwapd8l/zg7bnhZ4YPFVfa7quSpL86lfkPJSf4YuNBT+EvesPRP5vSqhl6vZVsQJwCR8alBooQ==", + "license": "CC-BY-4.0", + "dependencies": { + "colorbrewer": "1.5.6" + } + }, + "node_modules/charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, + "node_modules/clone": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", + "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/color": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/color/-/color-5.0.3.tgz", + "integrity": "sha512-ezmVcLR3xAVp8kYOm4GS45ZLLgIE6SPAFoduLr6hTDajwb3KZ2F46gulK3XpcwRFb5KKGCSezCBAY4Dw4HsyXA==", + "license": "MIT", + "peer": true, + "dependencies": { + "color-convert": "^3.1.3", + "color-string": "^2.1.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/color-convert": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-3.1.3.tgz", + "integrity": "sha512-fasDH2ont2GqF5HpyO4w0+BcewlhHEZOFn9c1ckZdHpJ56Qb7MHhH/IcJZbBGgvdtwdwNbLvxiBEdg336iA9Sg==", + "license": "MIT", + "peer": true, + "dependencies": { + "color-name": "^2.0.0" + }, + "engines": { + "node": ">=14.6" + } + }, + "node_modules/color-name": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-2.1.1.tgz", + "integrity": "sha512-p2FdgwVx1a9yWBHP2wI0VgShkDpgN4kZISkxdNipGBJWpa5G6b04OINlVWCyJj0JmfvcPrgqt95E9k8yvaOJFg==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12.20" + } + }, + "node_modules/color-string": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-2.1.4.tgz", + "integrity": "sha512-Bb6Cq8oq0IjDOe8wJmi4JeNn763Xs9cfrBcaylK1tPypWzyoy2G3l90v9k64kjphl/ZJjPIShFztenRomi8WTg==", + "license": "MIT", + "peer": true, + "dependencies": { + "color-name": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/colorbrewer": { + "version": "1.5.6", + "resolved": "https://registry.npmjs.org/colorbrewer/-/colorbrewer-1.5.6.tgz", + "integrity": "sha512-fONg2pGXyID8zNgKHBlagW8sb/AMShGzj4rRJfz5biZ7iuHQZYquSCLE/Co1oSQFmt/vvwjyezJCejQl7FG/tg==", + "license": [ + { + "type": "Apache-Style", + "url": "https://github.com/saikocat/colorbrewer/blob/master/LICENSE.txt" + } + ] + }, + "node_modules/commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 10" + } + }, + "node_modules/composed-offset-position": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/composed-offset-position/-/composed-offset-position-0.0.6.tgz", + "integrity": "sha512-Q7dLompI6lUwd7LWyIcP66r4WcS9u7AL2h8HaeipiRfCRPLMWqRx8fYsjb4OHi6UQFifO7XtNC2IlEJ1ozIFxw==", + "license": "MIT", + "peer": true, + "peerDependencies": { + "@floating-ui/utils": "^0.2.5" + } + }, + "node_modules/core-assert": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/core-assert/-/core-assert-0.2.1.tgz", + "integrity": "sha512-IG97qShIP+nrJCXMCgkNZgH7jZQ4n8RpPyPeXX++T6avR/KhLhgLiHKoEn5Rc1KjfycSfA9DMa6m+4C4eguHhw==", + "license": "MIT", + "dependencies": { + "buf-compare": "^1.0.0", + "is-error": "^2.2.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, + "node_modules/crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, + "node_modules/crypto-js": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz", + "integrity": "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==", + "deprecated": "Active development of CryptoJS has been discontinued. This library is no longer maintained.", + "license": "MIT", + "peer": true + }, + "node_modules/cssfilter": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/cssfilter/-/cssfilter-0.0.10.tgz", + "integrity": "sha512-FAaLDaplstoRsDR8XGYH51znUN0UY7nMc6Z9/fvE8EXGwvJE9hu7W2vHwx1+bd6gCYnln9nLbzxFTrcO9YQDZw==", + "license": "MIT", + "peer": true + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "license": "MIT", + "peer": true + }, + "node_modules/d3": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/d3/-/d3-7.9.0.tgz", + "integrity": "sha512-e1U46jVP+w7Iut8Jt8ri1YsPOvFpg46k+K8TpCb0P+zjCkjkPnV7WzfDJzMHy1LnA+wj5pLT1wjO901gLXeEhA==", + "license": "ISC", + "peer": true, + "dependencies": { + "d3-array": "3", + "d3-axis": "3", + "d3-brush": "3", + "d3-chord": "3", + "d3-color": "3", + "d3-contour": "4", + "d3-delaunay": "6", + "d3-dispatch": "3", + "d3-drag": "3", + "d3-dsv": "3", + "d3-ease": "3", + "d3-fetch": "3", + "d3-force": "3", + "d3-format": "3", + "d3-geo": "3", + "d3-hierarchy": "3", + "d3-interpolate": "3", + "d3-path": "3", + "d3-polygon": "3", + "d3-quadtree": "3", + "d3-random": "3", + "d3-scale": "4", + "d3-scale-chromatic": "3", + "d3-selection": "3", + "d3-shape": "3", + "d3-time": "3", + "d3-time-format": "4", + "d3-timer": "3", + "d3-transition": "3", + "d3-zoom": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "license": "ISC", + "dependencies": { + "internmap": "1 - 2" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-axis": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-axis/-/d3-axis-3.0.0.tgz", + "integrity": "sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw==", + "license": "ISC", + "peer": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-brush": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-brush/-/d3-brush-3.0.0.tgz", + "integrity": "sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ==", + "license": "ISC", + "peer": true, + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "3", + "d3-transition": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-chord": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-chord/-/d3-chord-3.0.1.tgz", + "integrity": "sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g==", + "license": "ISC", + "peer": true, + "dependencies": { + "d3-path": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-contour": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-contour/-/d3-contour-4.0.2.tgz", + "integrity": "sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA==", + "license": "ISC", + "peer": true, + "dependencies": { + "d3-array": "^3.2.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A==", + "license": "ISC", + "peer": true, + "dependencies": { + "delaunator": "5" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dispatch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", + "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==", + "license": "ISC", + "peer": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-drag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-3.0.0.tgz", + "integrity": "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==", + "license": "ISC", + "peer": true, + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-selection": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dsv": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz", + "integrity": "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==", + "license": "ISC", + "peer": true, + "dependencies": { + "commander": "7", + "iconv-lite": "0.6", + "rw": "1" + }, + "bin": { + "csv2json": "bin/dsv2json.js", + "csv2tsv": "bin/dsv2dsv.js", + "dsv2dsv": "bin/dsv2dsv.js", + "dsv2json": "bin/dsv2json.js", + "json2csv": "bin/json2dsv.js", + "json2dsv": "bin/json2dsv.js", + "json2tsv": "bin/json2dsv.js", + "tsv2csv": "bin/dsv2dsv.js", + "tsv2json": "bin/dsv2json.js" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", + "license": "BSD-3-Clause", + "peer": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-fetch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-fetch/-/d3-fetch-3.0.1.tgz", + "integrity": "sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw==", + "license": "ISC", + "peer": true, + "dependencies": { + "d3-dsv": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-force": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-force/-/d3-force-3.0.0.tgz", + "integrity": "sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg==", + "license": "ISC", + "peer": true, + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-quadtree": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-format": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.2.tgz", + "integrity": "sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-geo": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-3.1.1.tgz", + "integrity": "sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q==", + "license": "ISC", + "peer": true, + "dependencies": { + "d3-array": "2.5.0 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-hexbin": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/d3-hexbin/-/d3-hexbin-0.2.2.tgz", + "integrity": "sha512-KS3fUT2ReD4RlGCjvCEm1RgMtp2NFZumdMu4DBzQK8AZv3fXRM6Xm8I4fSU07UXvH4xxg03NwWKWdvxfS/yc4w==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-hierarchy": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz", + "integrity": "sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA==", + "license": "ISC", + "peer": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", + "license": "ISC", + "peer": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-polygon": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-polygon/-/d3-polygon-3.0.1.tgz", + "integrity": "sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg==", + "license": "ISC", + "peer": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-quadtree": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-3.0.1.tgz", + "integrity": "sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw==", + "license": "ISC", + "peer": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-random": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-random/-/d3-random-3.0.1.tgz", + "integrity": "sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ==", + "license": "ISC", + "peer": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-sankey": { + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/d3-sankey/-/d3-sankey-0.12.3.tgz", + "integrity": "sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ==", + "license": "BSD-3-Clause", + "peer": true, + "dependencies": { + "d3-array": "1 - 2", + "d3-shape": "^1.2.0" + } + }, + "node_modules/d3-sankey/node_modules/d3-array": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz", + "integrity": "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==", + "license": "BSD-3-Clause", + "peer": true, + "dependencies": { + "internmap": "^1.0.0" + } + }, + "node_modules/d3-sankey/node_modules/d3-path": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-1.0.9.tgz", + "integrity": "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==", + "license": "BSD-3-Clause", + "peer": true + }, + "node_modules/d3-sankey/node_modules/d3-shape": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-1.3.7.tgz", + "integrity": "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==", + "license": "BSD-3-Clause", + "peer": true, + "dependencies": { + "d3-path": "1" + } + }, + "node_modules/d3-sankey/node_modules/internmap": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-1.0.1.tgz", + "integrity": "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==", + "license": "ISC", + "peer": true + }, + "node_modules/d3-scale": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", + "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", + "license": "ISC", + "dependencies": { + "d3-array": "2.10.0 - 3", + "d3-format": "1 - 3", + "d3-interpolate": "1.2.0 - 3", + "d3-time": "2.1.1 - 3", + "d3-time-format": "2 - 4" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==", + "license": "ISC", + "peer": true, + "dependencies": { + "d3-color": "1 - 3", + "d3-interpolate": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-selection": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", + "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", + "license": "ISC", + "peer": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", + "license": "ISC", + "peer": true, + "dependencies": { + "d3-path": "^3.1.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", + "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", + "license": "ISC", + "dependencies": { + "d3-array": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time-format": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", + "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "license": "ISC", + "dependencies": { + "d3-time": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "license": "ISC", + "peer": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-transition": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-3.0.1.tgz", + "integrity": "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==", + "license": "ISC", + "peer": true, + "dependencies": { + "d3-color": "1 - 3", + "d3-dispatch": "1 - 3", + "d3-ease": "1 - 3", + "d3-interpolate": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + }, + "peerDependencies": { + "d3-selection": "2 - 3" + } + }, + "node_modules/d3-voronoi-map": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/d3-voronoi-map/-/d3-voronoi-map-2.1.1.tgz", + "integrity": "sha512-mCXfz/kD9IQxjHaU2IMjkO8fSo4J6oysPR2iL+omDsCy1i1Qn6BQ/e4hEAW8C6ms2kfuHwqtbNom80Hih94YsA==", + "license": "BSD-3-Clause", + "peer": true, + "dependencies": { + "d3-dispatch": "2.*", + "d3-polygon": "2.*", + "d3-timer": "2.*", + "d3-weighted-voronoi": "1.*" + } + }, + "node_modules/d3-voronoi-map/node_modules/d3-dispatch": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-2.0.0.tgz", + "integrity": "sha512-S/m2VsXI7gAti2pBoLClFFTMOO1HTtT0j99AuXLoGFKO6deHDdnv6ZGTxSTTUTgO1zVcv82fCOtDjYK4EECmWA==", + "license": "BSD-3-Clause", + "peer": true + }, + "node_modules/d3-voronoi-map/node_modules/d3-polygon": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/d3-polygon/-/d3-polygon-2.0.0.tgz", + "integrity": "sha512-MsexrCK38cTGermELs0cO1d79DcTsQRN7IWMJKczD/2kBjzNXxLUWP33qRF6VDpiLV/4EI4r6Gs0DAWQkE8pSQ==", + "license": "BSD-3-Clause", + "peer": true + }, + "node_modules/d3-voronoi-map/node_modules/d3-timer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-2.0.0.tgz", + "integrity": "sha512-TO4VLh0/420Y/9dO3+f9abDEFYeCUr2WZRlxJvbp4HPTQcSylXNiL6yZa9FIUvV1yRiFufl1bszTCLDqv9PWNA==", + "license": "BSD-3-Clause", + "peer": true + }, + "node_modules/d3-voronoi-treemap": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/d3-voronoi-treemap/-/d3-voronoi-treemap-1.1.2.tgz", + "integrity": "sha512-7odu9HdG/yLPWwzDteJq4yd9Q/NwgQV7IE/u36VQtcCK7k1sZwDqbkHCeMKNTBsq5mQjDwolTsrXcU0j8ZEMCA==", + "license": "BSD-3-Clause", + "peer": true, + "dependencies": { + "d3-voronoi-map": "2.*" + } + }, + "node_modules/d3-weighted-voronoi": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/d3-weighted-voronoi/-/d3-weighted-voronoi-1.1.3.tgz", + "integrity": "sha512-C3WdvSKl9aqhAy+f3QT3PPsQG6V+ajDfYO3BSclQDSD+araW2xDBFIH67aKzsSuuuKaX8K2y2dGq1fq/dWTVig==", + "license": "BSD-3-Clause", + "peer": true, + "dependencies": { + "d3-array": "2", + "d3-polygon": "2" + } + }, + "node_modules/d3-weighted-voronoi/node_modules/d3-array": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz", + "integrity": "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==", + "license": "BSD-3-Clause", + "peer": true, + "dependencies": { + "internmap": "^1.0.0" + } + }, + "node_modules/d3-weighted-voronoi/node_modules/d3-polygon": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/d3-polygon/-/d3-polygon-2.0.0.tgz", + "integrity": "sha512-MsexrCK38cTGermELs0cO1d79DcTsQRN7IWMJKczD/2kBjzNXxLUWP33qRF6VDpiLV/4EI4r6Gs0DAWQkE8pSQ==", + "license": "BSD-3-Clause", + "peer": true + }, + "node_modules/d3-weighted-voronoi/node_modules/internmap": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-1.0.1.tgz", + "integrity": "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==", + "license": "ISC", + "peer": true + }, + "node_modules/d3-zoom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-3.0.0.tgz", + "integrity": "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==", + "license": "ISC", + "peer": true, + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "2 - 3", + "d3-transition": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/deck.gl": { + "version": "9.3.10", + "resolved": "https://registry.npmjs.org/deck.gl/-/deck.gl-9.3.10.tgz", + "integrity": "sha512-iy+qGha/xj7isGwzF0eoDnF/VwzHQXyMIBb2pl3TBb8FXqzvheA/h4IKnsd7F4lji1SEeQ0A5kyqPmKnqTMgfQ==", + "license": "MIT", + "dependencies": { + "@deck.gl/aggregation-layers": "9.3.10", + "@deck.gl/arcgis": "9.3.10", + "@deck.gl/carto": "9.3.10", + "@deck.gl/core": "9.3.10", + "@deck.gl/extensions": "9.3.10", + "@deck.gl/geo-layers": "9.3.10", + "@deck.gl/google-maps": "9.3.10", + "@deck.gl/json": "9.3.10", + "@deck.gl/layers": "9.3.10", + "@deck.gl/mapbox": "9.3.10", + "@deck.gl/mesh-layers": "9.3.10", + "@deck.gl/react": "9.3.10", + "@deck.gl/widgets": "9.3.10", + "@loaders.gl/core": "^4.4.3", + "@luma.gl/core": "^9.3.5", + "@luma.gl/engine": "^9.3.5" + }, + "peerDependencies": { + "@arcgis/core": "^4.0.0", + "react": ">=16.3.0", + "react-dom": ">=16.3.0" + }, + "peerDependenciesMeta": { + "@arcgis/core": { + "optional": true + }, + "react": { + "optional": true + }, + "react-dom": { + "optional": true + } + } + }, + "node_modules/deep-equal": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.1.2.tgz", + "integrity": "sha512-5tdhKF6DbU7iIzrIOa1AOUt39ZRm13cmL1cGEh//aqR8x9+tNfbywRf0n5FD/18OKMdo7DNEtrX2t22ZAkI+eg==", + "license": "MIT", + "peer": true, + "dependencies": { + "is-arguments": "^1.1.1", + "is-date-object": "^1.0.5", + "is-regex": "^1.1.4", + "object-is": "^1.1.5", + "object-keys": "^1.1.1", + "regexp.prototype.flags": "^1.5.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/deep-strict-equal": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/deep-strict-equal/-/deep-strict-equal-0.2.0.tgz", + "integrity": "sha512-3daSWyvZ/zwJvuMGlzG1O+Ow0YSadGfb3jsh9xoCutv2tWyB9dA4YvR9L9/fSdDZa2dByYQe+TqapSGUrjnkoA==", + "license": "MIT", + "dependencies": { + "core-assert": "^0.2.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", + "peer": true, + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/define-properties": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", + "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", + "license": "MIT", + "peer": true, + "dependencies": { + "define-data-property": "^1.0.1", + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/delaunator": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/delaunator/-/delaunator-5.1.0.tgz", + "integrity": "sha512-AGrQ4QSgssa1NGmWmLPqN5NY2KajF5MqxetNEO+o0n3ZwZZeTmt7bBnvzHWrmkZFxGgr4HdyFgelzgi06otLuQ==", + "license": "ISC", + "peer": true, + "dependencies": { + "robust-predicates": "^3.0.2" + } + }, + "node_modules/dfa": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/dfa/-/dfa-1.2.0.tgz", + "integrity": "sha512-ED3jP8saaweFTjeGX8HQPjeC1YYyZs98jGNZx6IiBvxW7JG5v492kamAQB3m2wop07CvU/RQmzcKr6bgcC5D/Q==", + "license": "MIT", + "peer": true + }, + "node_modules/draco3d": { + "version": "1.5.7", + "resolved": "https://registry.npmjs.org/draco3d/-/draco3d-1.5.7.tgz", + "integrity": "sha512-m6WCKt/erDXcw+70IJXnG7M3awwQPAsZvJGX5zY7beBqpELw6RDGkYVU0W43AFxye4pDZ5i2Lbyc/NNGqwjUVQ==", + "license": "Apache-2.0" + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "peer": true, + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/earcut": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/earcut/-/earcut-2.2.4.tgz", + "integrity": "sha512-/pjZsA1b4RPHbeWZQn66SWS8nZZWLQQ23oE3Eam7aroEFGEvwKAsJfZ9ytiEMycfzXWpca4FA9QIOehf7PocBQ==", + "license": "ISC" + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "peer": true, + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-toolkit": { + "version": "1.50.0", + "resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.50.0.tgz", + "integrity": "sha512-OyZKhUVvEep9ITEiwHn8GKnMRQIVqoSIX7WnRbkWgJkllCujilqP2rD0u979tkl8wqyc8ICwlc1UBVv/Sl1G6w==", + "license": "MIT", + "peer": true, + "workspaces": [ + "docs", + "benchmarks", + "tests/types" + ] + }, + "node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/esri-loader": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/esri-loader/-/esri-loader-3.7.0.tgz", + "integrity": "sha512-cB1Sw9EQjtW4mtT7eFBjn/6VaaIWNTjmTd2asnnEyuZk1xVSFRMCfLZSBSjZM7ZarDcVu5WIjOP0t0MYVu4hVQ==", + "deprecated": "Use @arcgis/core instead.", + "license": "Apache-2.0" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz", + "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/fast-xml-builder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.0.tgz", + "integrity": "sha512-F74cZEdCvuw9P41GAC3rod4X04jjWGM1JPEv/GWSqFTWLsdyMSBMBMlm9Hk3GLBgLBbdBNY8yee0pQh2RBVESQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.6.2", + "xml-naming": "^0.3.0" + } + }, + "node_modules/fast-xml-parser": { + "version": "5.10.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.10.1.tgz", + "integrity": "sha512-IEMIf7298kXuZSRFoGfMYrl7is8LpavODgbNz1cwIudv7KwVFnuU+UsMporfq6PD6aXSlawZlARiA3UywCTfMw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^3.0.0", + "fast-xml-builder": "^1.2.0", + "is-unsafe": "^2.0.0", + "path-expression-matcher": "^1.6.2", + "strnum": "^2.4.1", + "xml-naming": "^0.3.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fflate": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.3.tgz", + "integrity": "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==", + "license": "MIT" + }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0" + }, + "node_modules/flatpickr": { + "version": "4.6.13", + "resolved": "https://registry.npmjs.org/flatpickr/-/flatpickr-4.6.13.tgz", + "integrity": "sha512-97PMG/aywoYpB4IvbvUJi0RQi8vearvU0oov1WW3k0WZPBMrTQVqekSX5CjSG/M4Q3i6A/0FKXC7RyAoAUUSPw==", + "license": "MIT", + "peer": true + }, + "node_modules/focus-trap": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/focus-trap/-/focus-trap-7.8.0.tgz", + "integrity": "sha512-/yNdlIkpWbM0ptxno3ONTuf+2g318kh2ez3KSeZN5dZ8YC6AAmgeWz+GasYYiBJPFaYcSAPeu4GfhUaChzIJXA==", + "license": "MIT", + "peer": true, + "dependencies": { + "tabbable": "^6.4.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "peer": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/functions-have-names": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", + "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", + "license": "MIT", + "peer": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "peer": true, + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gl-matrix": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/gl-matrix/-/gl-matrix-3.4.4.tgz", + "integrity": "sha512-latSnyDNt/8zYUB6VIJ6PCh2jBjJX6gnDsoCZ7LyW7GkqrD51EWwa9qCoGixj8YqBtETQK/xY7OmpTF8xz1DdQ==", + "license": "MIT" + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/h3-js": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/h3-js/-/h3-js-4.5.0.tgz", + "integrity": "sha512-uKmdPc+DuarnR+XqZxEuWOMw7KzzKROrx3MLeJpFnMOs78S9M5eZ+X5RieS9UcSFQqbeXzbfWuX/W9Pyajinqw==", + "license": "Apache-2.0", + "engines": { + "node": ">=4", + "npm": ">=3", + "yarn": ">=1.3.0" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", + "peer": true, + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "peer": true, + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "peer": true, + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "peer": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/image-size": { + "version": "0.7.5", + "resolved": "https://registry.npmjs.org/image-size/-/image-size-0.7.5.tgz", + "integrity": "sha512-Hiyv+mXHfFEP7LzUL/llg9RwFxxY+o9N3JVLIeG5E7iFIFAalxvRU9UZthBdYDEVnzHMgjnKJPPpay5BWf1g9g==", + "license": "MIT", + "bin": { + "image-size": "bin/image-size.js" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "license": "MIT" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/interactjs": { + "version": "1.10.28", + "resolved": "https://registry.npmjs.org/interactjs/-/interactjs-1.10.28.tgz", + "integrity": "sha512-QCa4ksTPd2p/FZ4I6hrH6fogjLqru1TxYywRiKYNuLZtrlAxTIYUa/MYsE7PnUgWZJU3SrGVDjiz065BeaXYng==", + "license": "MIT", + "peer": true, + "dependencies": { + "@interactjs/types": "1.10.28" + } + }, + "node_modules/internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/is-arguments": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.2.0.tgz", + "integrity": "sha512-7bVbi0huj/wrIAOzb8U1aszg9kdi3KN/CyU19CTI7tAoZYEZoL9yCDXpbXN+uPsuWnP02cyug1gleqq+TU+YCA==", + "license": "MIT", + "peer": true, + "dependencies": { + "call-bound": "^1.0.2", + "has-tostringtag": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "license": "MIT" + }, + "node_modules/is-date-object": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.1.0.tgz", + "integrity": "sha512-PwwhEakHVKTdRNVOw+/Gyh0+MzlCl4R6qKvkhuvLtPMggI1WAHt9sOwZxQLSGpUaDnrdyDsomoRgNnCfKNSXXg==", + "license": "MIT", + "peer": true, + "dependencies": { + "call-bound": "^1.0.2", + "has-tostringtag": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-error": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/is-error/-/is-error-2.2.2.tgz", + "integrity": "sha512-IOQqts/aHWbiisY5DuPJQ0gcbvaLFCa7fBa9xoLfxBZvQ+ZI/Zh9xoI7Gk+G64N0FdK4AbibytHht2tWgpJWLg==", + "license": "MIT" + }, + "node_modules/is-regex": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", + "integrity": "sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==", + "license": "MIT", + "peer": true, + "dependencies": { + "call-bound": "^1.0.2", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-unsafe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.0.tgz", + "integrity": "sha512-2LdV822R+wmI86unXA93WCFpL6g+av8ynWk0nrHyJqGop5VoocYsSLFgN8jrfalT6iGeLNM4KXuVSsULP53kEA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/jpeg-exif": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/jpeg-exif/-/jpeg-exif-1.1.4.tgz", + "integrity": "sha512-a+bKEcCjtuW5WTdgeXFzswSrdqi0jk4XlEtZlx5A94wCoBpFjfFTbo/Tra5SpNCl/YFZPvcV1dJc+TAYeg6ROQ==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT", + "peer": true + }, + "node_modules/jsep": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/jsep/-/jsep-1.4.0.tgz", + "integrity": "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==", + "license": "MIT", + "engines": { + "node": ">= 10.16.0" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/json-with-bigint": { + "version": "3.5.10", + "resolved": "https://registry.npmjs.org/json-with-bigint/-/json-with-bigint-3.5.10.tgz", + "integrity": "sha512-Vcx+JVNEBts/xfcoCS69sKrOhOk/3TVlvlT+XzUOefVKnnrbYSCKpDCm10pohsJFtsJVYnwa/cXRZ4eElzaM6w==", + "license": "MIT" + }, + "node_modules/jszip": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", + "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", + "license": "(MIT OR GPL-3.0-or-later)", + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, + "node_modules/ktx-parse": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/ktx-parse/-/ktx-parse-0.7.1.tgz", + "integrity": "sha512-FeA3g56ksdFNwjXJJsc1CCc7co+AJYDp6ipIp878zZ2bU8kWROatLYf39TQEd4/XRSUvBXovQ8gaVKWPXsCLEQ==", + "license": "MIT" + }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "license": "MIT", + "dependencies": { + "immediate": "~3.0.5" + } + }, + "node_modules/lit": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/lit/-/lit-3.3.3.tgz", + "integrity": "sha512-fycuvZg/hkpozL00lm1pEJH5nN/lr9ZXd6mJI2HSN4+Bzc+LDNdEApJ6HFbPkdFNHLvOplIIuJvxkS4XUxqirw==", + "license": "BSD-3-Clause", + "peer": true, + "dependencies": { + "@lit/reactive-element": "^2.1.0", + "lit-element": "^4.2.0", + "lit-html": "^3.3.0" + } + }, + "node_modules/lit-element": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/lit-element/-/lit-element-4.2.2.tgz", + "integrity": "sha512-aFKhNToWxoyhkNDmWZwEva2SlQia+jfG0fjIWV//YeTaWrVnOxD89dPKfigCUspXFmjzOEUQpOkejH5Ly6sG0w==", + "license": "BSD-3-Clause", + "peer": true, + "dependencies": { + "@lit-labs/ssr-dom-shim": "^1.5.0", + "@lit/reactive-element": "^2.1.0", + "lit-html": "^3.3.0" + } + }, + "node_modules/lit-html": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/lit-html/-/lit-html-3.3.3.tgz", + "integrity": "sha512-el8M6jK2o3RXBnrSHX3ZKrsN8zEV63pSExTO1wYJz7QndGYZ8353e2a5PPX+qHe2aGayfnchQmkAojaWAREOIA==", + "license": "BSD-3-Clause", + "peer": true, + "dependencies": { + "@types/trusted-types": "^2.0.2" + } + }, + "node_modules/long": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/long/-/long-3.2.0.tgz", + "integrity": "sha512-ZYvPPOMqUwPoDsbJaR10iQJYnMuZhRTvHYl62ErLIEX7RgFlziSBUUvrt3OVfc47QlHHpzPZYP17g3Fv7oeJkg==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/lz4js": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/lz4js/-/lz4js-0.2.0.tgz", + "integrity": "sha512-gY2Ia9Lm7Ep8qMiuGRhvUq0Q7qUereeldZPP1PMEJxPtEWHJLqw9pgX68oHajBH0nzJK4MaZEA/YNV3jT8u8Bg==", + "license": "ISC", + "optional": true + }, + "node_modules/marked": { + "version": "16.3.0", + "resolved": "https://registry.npmjs.org/marked/-/marked-16.3.0.tgz", + "integrity": "sha512-K3UxuKu6l6bmA5FUwYho8CfJBlsUWAooKtdGgMcERSpF7gcBUrCGsLH7wDaaNOzwq18JzSUDyoEb/YsrqMac3w==", + "license": "MIT", + "peer": true, + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/markerjs2": { + "version": "2.32.7", + "resolved": "https://registry.npmjs.org/markerjs2/-/markerjs2-2.32.7.tgz", + "integrity": "sha512-HeFRZjmc43DOG3lSQp92z49cq2oCYpYn2pX++SkJAW1Dij4xJtRquVRf+cXeSZQWDX3ufns1Ry/bGk+zveP7rA==", + "license": "SEE LICENSE IN LICENSE", + "peer": true + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "license": "BSD-3-Clause", + "dependencies": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + } + }, + "node_modules/mjolnir.js": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mjolnir.js/-/mjolnir.js-3.0.1.tgz", + "integrity": "sha512-/RMi8Jm3NKleOkVI8D2ai+1OVwtfRJsSVBVjbTXNm83nfsN4uORYaN3u1/hsg5CqVI+di8enTkvgDNDOywn6cQ==", + "license": "MIT" + }, + "node_modules/moment": { + "version": "2.30.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", + "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/moment-timezone": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.6.3.tgz", + "integrity": "sha512-pVEPA/HCFHHbwJ130ywnzYuZpkEGcP6Daa/OwNebpA18MybeFHmQilAGGovXgWijQ8vQtmud9jZrziUBgsykfg==", + "license": "MIT", + "dependencies": { + "moment": "^2.29.4" + }, + "engines": { + "node": "*" + } + }, + "node_modules/nanoid": { + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/object-is": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.6.tgz", + "integrity": "sha512-F8cZ+KfGlSGi09lJT7/Nd6KJZ9ygtvYC0/UYYLI9nmQKLMnydpB9yvbv9K1uSkEu7FU9vYPmVwLg328tX+ot3Q==", + "license": "MIT", + "peer": true, + "dependencies": { + "call-bind": "^1.0.7", + "define-properties": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, + "node_modules/path-expression-matcher": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", + "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/pbf": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/pbf/-/pbf-3.3.0.tgz", + "integrity": "sha512-XDF38WCH3z5OV/OVa8GKUNtLAyneuzbCisx7QUCF8Q6Nutx0WnJrQe5O+kOtBlLfRNUws98Y58Lblp+NJG5T4Q==", + "license": "BSD-3-Clause", + "dependencies": { + "ieee754": "^1.1.12", + "resolve-protobuf-schema": "^2.1.0" + }, + "bin": { + "pbf": "bin/pbf" + } + }, + "node_modules/pdfmake": { + "version": "0.2.23", + "resolved": "https://registry.npmjs.org/pdfmake/-/pdfmake-0.2.23.tgz", + "integrity": "sha512-A/IksoKb/ikOZH1edSDJ/2zBbqJKDghD4+fXn3rT7quvCJDlsZMs3NmIB3eajLMMFU9Bd3bZPVvlUMXhvFI+bQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "@foliojs-fork/linebreak": "^1.1.2", + "@foliojs-fork/pdfkit": "^0.15.3", + "iconv-lite": "^0.7.1", + "xmldoc": "^2.0.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/pdfmake/node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/png-js": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/png-js/-/png-js-1.1.0.tgz", + "integrity": "sha512-PM/uYGzGdNSzqeOgly68+6wKQDL1SY0a/N+OEa/+br6LnHWOAJB0Npiamnodfq3jd2LS/i2fMeOKSAILjA+m5Q==", + "peer": true, + "dependencies": { + "browserify-zlib": "^0.2.0" + } + }, + "node_modules/polylabel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/polylabel/-/polylabel-1.1.0.tgz", + "integrity": "sha512-bxaGcA40sL3d6M4hH72Z4NdLqxpXRsCFk8AITYg6x1rn1Ei3izf00UMLklerBZTO49aPA3CYrIwVulx2Bce2pA==", + "license": "ISC", + "peer": true, + "dependencies": { + "tinyqueue": "^2.0.3" + } + }, + "node_modules/postcss": { + "version": "8.5.26", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", + "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.17", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/preact": { + "version": "10.29.8", + "resolved": "https://registry.npmjs.org/preact/-/preact-10.29.8.tgz", + "integrity": "sha512-ej2aVZ+vZ8WO7tvlQWRM9N63A0KzF9q4mWJfDUHgYaIofWY9hu74QdnQrjoPMmZi2/nZ5gN0bJCQF49xQqx09Q==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/preact" + }, + "peerDependencies": { + "preact-render-to-string": ">=5" + }, + "peerDependenciesMeta": { + "preact-render-to-string": { + "optional": true + } + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, + "node_modules/protocol-buffers-schema": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/protocol-buffers-schema/-/protocol-buffers-schema-3.6.1.tgz", + "integrity": "sha512-VG2K63Igkiv9p76tk1lilczEK1cT+kCjKtkdhw1dQZV3k3IXJbd3o6Ho8b9zJZaHSnT2hKe4I+ObmX9w6m5SmQ==", + "license": "MIT" + }, + "node_modules/quadbin": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/quadbin/-/quadbin-0.4.2.tgz", + "integrity": "sha512-1NFzjFVM23Um51/ttD6lFDqGtUHNS5Ky1slZHk3YPwMbC+7Jl3ULLb4QvDo6+Nerv8b8SgUV+ysOhziUh4B5cQ==", + "license": "MIT", + "dependencies": { + "@math.gl/web-mercator": "^4.1.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/react": { + "version": "19.2.8", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.8.tgz", + "integrity": "sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.2.8", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.8.tgz", + "integrity": "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "scheduler": "^0.27.0" + }, + "peerDependencies": { + "react": "^19.2.8" + } + }, + "node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/regexp.prototype.flags": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz", + "integrity": "sha512-dYqgNSZbDwkaJ2ceRd9ojCGjBq+mOm9LmtXnAnEGyHhN/5R7iDW2TRw3h+o/jCFxus3P2LfWIIiwowAjANm7IA==", + "license": "MIT", + "peer": true, + "dependencies": { + "call-bind": "^1.0.8", + "define-properties": "^1.2.1", + "es-errors": "^1.3.0", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "set-function-name": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve-protobuf-schema": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/resolve-protobuf-schema/-/resolve-protobuf-schema-2.1.0.tgz", + "integrity": "sha512-kI5ffTiZWmJaS/huM8wZfEMer1eRd7oJQhDuxeCLe3t7N7mX3z94CN0xPxBQxFYQTSNz9T0i+v6inKqSdK8xrQ==", + "license": "MIT", + "dependencies": { + "protocol-buffers-schema": "^3.3.1" + } + }, + "node_modules/robust-predicates": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.3.tgz", + "integrity": "sha512-NS3levdsRIUOmiJ8FZWCP7LG3QpJyrs/TE0Zpf1yvZu8cAJJ6QMW92H1c7kWpdIHo8RvmLxN/o2JXTKHp74lUA==", + "license": "Unlicense", + "peer": true + }, + "node_modules/rollup": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.62.4.tgz", + "integrity": "sha512-RXOqwaPsBGjMNMa4sQjDjHieHEZDFoj/Rdr46l2MU5DfEs16wHJPC2RPTPHWhNl+M3aI472LLqFkFKut4SblOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.9" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@napi-rs/lzma-linux-x64-gnu": "1.5.1", + "@rollup/rollup-android-arm-eabi": "4.62.4", + "@rollup/rollup-android-arm64": "4.62.4", + "@rollup/rollup-darwin-arm64": "4.62.4", + "@rollup/rollup-darwin-x64": "4.62.4", + "@rollup/rollup-freebsd-arm64": "4.62.4", + "@rollup/rollup-freebsd-x64": "4.62.4", + "@rollup/rollup-linux-arm-gnueabihf": "4.62.4", + "@rollup/rollup-linux-arm-musleabihf": "4.62.4", + "@rollup/rollup-linux-arm64-gnu": "4.62.4", + "@rollup/rollup-linux-arm64-musl": "4.62.4", + "@rollup/rollup-linux-loong64-gnu": "4.62.4", + "@rollup/rollup-linux-loong64-musl": "4.62.4", + "@rollup/rollup-linux-ppc64-gnu": "4.62.4", + "@rollup/rollup-linux-ppc64-musl": "4.62.4", + "@rollup/rollup-linux-riscv64-gnu": "4.62.4", + "@rollup/rollup-linux-riscv64-musl": "4.62.4", + "@rollup/rollup-linux-s390x-gnu": "4.62.4", + "@rollup/rollup-linux-x64-gnu": "4.62.4", + "@rollup/rollup-linux-x64-musl": "4.62.4", + "@rollup/rollup-openbsd-x64": "4.62.4", + "@rollup/rollup-openharmony-arm64": "4.62.4", + "@rollup/rollup-win32-arm64-msvc": "4.62.4", + "@rollup/rollup-win32-ia32-msvc": "4.62.4", + "@rollup/rollup-win32-x64-gnu": "4.62.4", + "@rollup/rollup-win32-x64-msvc": "4.62.4", + "fsevents": "~2.3.2" + } + }, + "node_modules/rw": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", + "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==", + "license": "BSD-3-Clause", + "peer": true + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT", + "peer": true + }, + "node_modules/sax": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.1.tgz", + "integrity": "sha512-42tBVwLWnaQvW5zc4HbZrTuWccECCZfBi92FDuwtqxasH+JbPB3/FOKb1m222K42R4WxuxzzMsTswfzgtSu64Q==", + "license": "BlueOak-1.0.0", + "peer": true, + "engines": { + "node": ">=11.0.0" + } + }, + "node_modules/scheduler": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", + "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", + "license": "MIT", + "peer": true + }, + "node_modules/seedrandom": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==", + "license": "MIT", + "peer": true + }, + "node_modules/set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "license": "MIT", + "peer": true, + "dependencies": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/set-function-name": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/set-function-name/-/set-function-name-2.0.2.tgz", + "integrity": "sha512-7PGFlmtwsEADb0WYyvCMa1t+yke6daIG4Wirafur5kcf+MhUnPms1UeR0CKQdTZD81yESwMHbtn+TR+dMviakQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "functions-have-names": "^1.2.3", + "has-property-descriptors": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "license": "MIT" + }, + "node_modules/snappyjs": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/snappyjs/-/snappyjs-0.6.1.tgz", + "integrity": "sha512-YIK6I2lsH072UE0aOFxxY1dPDCS43I5ktqHpeAsuLNYWkE5pGxRGWfDM4/vSUfNzXjC1Ivzt3qx31PCLmc9yqg==", + "license": "MIT" + }, + "node_modules/sortablejs": { + "version": "1.15.7", + "resolved": "https://registry.npmjs.org/sortablejs/-/sortablejs-1.15.7.tgz", + "integrity": "sha512-Kk8wLQPlS+yi1ZEf48a4+fzHa4yxjC30M/Sr2AnQu+f/MPwvvX9XjZ6OWejiz8crBsLwSq8GHqaxaET7u6ux0A==", + "license": "MIT", + "peer": true + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "license": "BSD-3-Clause" + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/strnum": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.1.tgz", + "integrity": "sha512-M9eUSMT2dCB2cTNPG7UYj6KuK7RJR2SN2+yCV/fTW3xzTCS6EaGZ5pSMgDIjB7r8zSfTGk+dvvn9rTjpVS9Mwg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "anynum": "^1.0.1" + } + }, + "node_modules/svg-arc-to-cubic-bezier": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/svg-arc-to-cubic-bezier/-/svg-arc-to-cubic-bezier-3.2.0.tgz", + "integrity": "sha512-djbJ/vZKZO+gPoSDThGNpKDO+o+bAeA4XQKovvkNCqnIS2t+S4qnLAGQhyyrulhCFRl1WWzAp0wUDV8PpTVU3g==", + "license": "ISC", + "peer": true + }, + "node_modules/tabbable": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.5.0.tgz", + "integrity": "sha512-wieBHXygIm7OyQOu5hQlkk62/WyCFYGlWg7L6/ZCUZwx0o398Zkn4pVmMyfYhfMG8kGrj/Krt8eIk6UKC6VzwA==", + "license": "MIT", + "peer": true + }, + "node_modules/texture-compressor": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/texture-compressor/-/texture-compressor-1.0.2.tgz", + "integrity": "sha512-dStVgoaQ11mA5htJ+RzZ51ZxIZqNOgWKAIvtjLrW1AliQQLCmrDqNzQZ8Jh91YealQ95DXt4MEduLzJmbs6lig==", + "license": "MIT", + "dependencies": { + "argparse": "^1.0.10", + "image-size": "^0.7.4" + }, + "bin": { + "texture-compressor": "bin/texture-compressor.js" + } + }, + "node_modules/timezone-groups": { + "version": "0.10.4", + "resolved": "https://registry.npmjs.org/timezone-groups/-/timezone-groups-0.10.4.tgz", + "integrity": "sha512-AnkJYrbb7uPkDCEqGeVJiawZNiwVlSkkeX4jZg1gTEguClhyX+/Ezn07KB6DT29tG3UN418ldmS/W6KqGOTDjg==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=18.12.0" + } + }, + "node_modules/tiny-inflate": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/tiny-inflate/-/tiny-inflate-1.0.3.tgz", + "integrity": "sha512-pkY1fj1cKHb2seWDy0B16HeWyczlJA9/WW3u3c4z/NiWDsO3DOU5D7nhTLE9CF0yXv/QZFY7sEJmj24dK+Rrqw==", + "license": "MIT", + "peer": true + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyqueue": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/tinyqueue/-/tinyqueue-2.0.3.tgz", + "integrity": "sha512-ppJZNDuKGgxzkHihX8v9v9G5f+18gzaTfrukGrq6ueg0lmH4nqVnA2IPG0AEH3jKEk2GRJCUhDoqpoiw3PHLBA==", + "license": "ISC", + "peer": true + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/type-fest": { + "version": "4.41.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", + "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", + "license": "(MIT OR CC0-1.0)", + "peer": true, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "devOptional": true, + "license": "MIT" + }, + "node_modules/unicode-properties": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/unicode-properties/-/unicode-properties-1.4.1.tgz", + "integrity": "sha512-CLjCCLQ6UuMxWnbIylkisbRj31qxHPAurvena/0iwSVbQ2G1VY5/HjV0IRabOEbDHlzZlRdCrD4NhB0JtU40Pg==", + "license": "MIT", + "peer": true, + "dependencies": { + "base64-js": "^1.3.0", + "unicode-trie": "^2.0.0" + } + }, + "node_modules/unicode-trie": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-trie/-/unicode-trie-2.0.0.tgz", + "integrity": "sha512-x7bc76x0bm4prf1VLg79uhAzKw8DVboClSN5VxJuQ+LKDOVEW9CdH+VY7SP+vX7xCYQqzzgQpFqz15zeLvAtZQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "pako": "^0.2.5", + "tiny-inflate": "^1.0.0" + } + }, + "node_modules/unicode-trie/node_modules/pako": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz", + "integrity": "sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA==", + "license": "MIT", + "peer": true + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/vite": { + "version": "6.4.3", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.3.tgz", + "integrity": "sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.25.0", + "fdir": "^6.4.4", + "picomatch": "^4.0.2", + "postcss": "^8.5.3", + "rollup": "^4.34.9", + "tinyglobby": "^0.2.13" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "jiti": ">=1.21.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/ws": { + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", + "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xml-naming": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", + "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/xmldoc": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/xmldoc/-/xmldoc-2.0.3.tgz", + "integrity": "sha512-6gRk4NY/Jvg67xn7OzJuxLRsGgiXBaPUQplVJ/9l99uIugxh4FTOewYz5ic8WScj7Xx/2WvhENiQKwkK9RpE4w==", + "license": "MIT", + "peer": true, + "dependencies": { + "sax": "^1.4.3" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/xss": { + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/xss/-/xss-1.0.13.tgz", + "integrity": "sha512-clu7dxTm1e8Mo5fz3n/oW3UCXBfV89xZ72jM8yzo1vR/pIS0w3sgB3XV2H8Vm6zfGnHL0FzvLJPJEBhd86/z4Q==", + "license": "MIT", + "peer": true, + "dependencies": { + "commander": "^2.20.3", + "cssfilter": "0.0.10" + }, + "bin": { + "xss": "bin/xss" + }, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/xss/node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "license": "MIT", + "peer": true + }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, + "node_modules/zstd-codec": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/zstd-codec/-/zstd-codec-0.1.5.tgz", + "integrity": "sha512-v3fyjpK8S/dpY/X5WxqTK3IoCnp/ZOLxn144GZVlNUjtwAchzrVo03h+oMATFhCIiJ5KTr4V3vDQQYz4RU684g==", + "license": "MIT", + "optional": true + }, + "packages/api": { + "name": "@pact-tailor/api", + "version": "0.1.0", + "dependencies": { + "@pact-tailor/graph": "*", + "@pact-tailor/intel": "*", + "@pact-tailor/ontology": "*", + "@pact-tailor/registry": "*", + "@pact-tailor/state": "*", + "@pact-tailor/store": "*" + }, + "engines": { + "node": ">=20" + } + }, + "packages/connectors": { + "name": "@pact-tailor/connectors", + "version": "0.1.0", + "dependencies": { + "@pact-tailor/ontology": "*", + "@pact-tailor/registry": "*", + "@pact-tailor/store": "*", + "fflate": "^0.8.2", + "ws": "^8.18.0" + }, + "devDependencies": { + "@types/ws": "^8.5.13" + }, + "engines": { + "node": ">=20" + } + }, + "packages/globe": { + "name": "@pact-tailor/globe", + "version": "0.1.0", + "dependencies": { + "deck.gl": "^9.0.38" + }, + "devDependencies": { + "typescript": "^5.7.2", + "vite": "^6.0.5" + }, + "engines": { + "node": ">=20" + } + }, + "packages/graph": { + "name": "@pact-tailor/graph", + "version": "0.1.0", + "dependencies": { + "@pact-tailor/ontology": "*" + }, + "engines": { + "node": ">=20" + } + }, + "packages/intel": { + "name": "@pact-tailor/intel", + "version": "0.1.0", + "dependencies": { + "@pact-tailor/graph": "*", + "@pact-tailor/ontology": "*", + "@pact-tailor/state": "*" + }, + "engines": { + "node": ">=20" + } + }, + "packages/ontology": { + "name": "@pact-tailor/ontology", + "version": "0.1.0", + "dependencies": { + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1" + }, + "engines": { + "node": ">=20" + } + }, + "packages/pact-bridge": { + "name": "@pact-tailor/pact-bridge", + "version": "0.1.0", + "dependencies": { + "@pact-tailor/intel": "*", + "@pact-tailor/ontology": "*" + }, + "engines": { + "node": ">=20" + } + }, + "packages/registry": { + "name": "@pact-tailor/registry", + "version": "0.1.0", + "dependencies": { + "@pact-tailor/ontology": "*", + "yaml": "^2.6.1" + }, + "engines": { + "node": ">=20" + } + }, + "packages/state": { + "name": "@pact-tailor/state", + "version": "0.1.0", + "dependencies": { + "@pact-tailor/ontology": "*", + "@pact-tailor/store": "*" + }, + "engines": { + "node": ">=20" + } + }, + "packages/store": { + "name": "@pact-tailor/store", + "version": "0.1.0", + "dependencies": { + "@pact-tailor/ontology": "*" + }, + "engines": { + "node": ">=20" + } + } + } +} diff --git a/intelligence/package.json b/intelligence/package.json new file mode 100644 index 0000000..e36557c --- /dev/null +++ b/intelligence/package.json @@ -0,0 +1,25 @@ +{ + "name": "@pact-tailor/intelligence-workspace", + "version": "0.1.0", + "private": true, + "description": "PACT.TAILOR intelligence network — Tailor extension workspace: real-time energy, industry & trade intelligence over the PACT protocol", + "type": "module", + "workspaces": [ + "packages/*" + ], + "engines": { + "node": ">=20" + }, + "scripts": { + "build": "npm run build -w @pact-tailor/ontology && npm run build -w @pact-tailor/registry && npm run build -w @pact-tailor/store && npm run build -w @pact-tailor/graph && npm run build -w @pact-tailor/state && npm run build -w @pact-tailor/connectors && npm run build -w @pact-tailor/intel && npm run build -w @pact-tailor/pact-bridge && npm run build -w @pact-tailor/api", + "build:globe": "npm run build -w @pact-tailor/globe", + "test": "npm test --workspaces --if-present", + "validate:data": "node packages/registry/scripts/validate-data.mjs", + "verify:live": "node packages/connectors/dist/verify-live.js", + "smoke": "bash scripts/smoke.sh" + }, + "devDependencies": { + "@types/node": "^22.10.0", + "typescript": "^5.7.2" + } +} diff --git a/intelligence/packages/api/package.json b/intelligence/packages/api/package.json new file mode 100644 index 0000000..0c05044 --- /dev/null +++ b/intelligence/packages/api/package.json @@ -0,0 +1,26 @@ +{ + "name": "@pact-tailor/api", + "version": "0.1.0", + "private": true, + "description": "Zero-dependency REST + SSE API over the knowledge graph and real-time state, with server-driven UI (SDUI) composition", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": "./dist/index.js" + }, + "engines": { + "node": ">=20" + }, + "scripts": { + "build": "tsc -p tsconfig.json" + }, + "dependencies": { + "@pact-tailor/graph": "*", + "@pact-tailor/intel": "*", + "@pact-tailor/ontology": "*", + "@pact-tailor/registry": "*", + "@pact-tailor/state": "*", + "@pact-tailor/store": "*" + } +} diff --git a/intelligence/packages/api/src/index.ts b/intelligence/packages/api/src/index.ts new file mode 100644 index 0000000..60dcd6d --- /dev/null +++ b/intelligence/packages/api/src/index.ts @@ -0,0 +1,2 @@ +// @pact-tailor/api — REST + SSE server + SDUI composer. Populated in the API commit. +export {}; diff --git a/intelligence/packages/api/tsconfig.json b/intelligence/packages/api/tsconfig.json new file mode 100644 index 0000000..5a24989 --- /dev/null +++ b/intelligence/packages/api/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/intelligence/packages/connectors/package.json b/intelligence/packages/connectors/package.json new file mode 100644 index 0000000..ad2dc64 --- /dev/null +++ b/intelligence/packages/connectors/package.json @@ -0,0 +1,28 @@ +{ + "name": "@pact-tailor/connectors", + "version": "0.1.0", + "private": true, + "description": "Connector SDK (discover/fetch/parse/normalize/verify, replay|live HttpGate) + AEMO, AIS, BOM, ENTSO-E, EIA connectors and fixtures", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": "./dist/index.js" + }, + "engines": { + "node": ">=20" + }, + "scripts": { + "build": "tsc -p tsconfig.json" + }, + "dependencies": { + "@pact-tailor/ontology": "*", + "@pact-tailor/registry": "*", + "@pact-tailor/store": "*", + "fflate": "^0.8.2", + "ws": "^8.18.0" + }, + "devDependencies": { + "@types/ws": "^8.5.13" + } +} diff --git a/intelligence/packages/connectors/src/index.ts b/intelligence/packages/connectors/src/index.ts new file mode 100644 index 0000000..83e8fa8 --- /dev/null +++ b/intelligence/packages/connectors/src/index.ts @@ -0,0 +1,2 @@ +// @pact-tailor/connectors — connector SDK and connectors. Populated in the connector commit. +export {}; diff --git a/intelligence/packages/connectors/tsconfig.json b/intelligence/packages/connectors/tsconfig.json new file mode 100644 index 0000000..5a24989 --- /dev/null +++ b/intelligence/packages/connectors/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/intelligence/packages/globe/package.json b/intelligence/packages/globe/package.json new file mode 100644 index 0000000..20aec72 --- /dev/null +++ b/intelligence/packages/globe/package.json @@ -0,0 +1,22 @@ +{ + "name": "@pact-tailor/globe", + "version": "0.1.0", + "private": true, + "description": "GPU-rendered interactive globe (deck.gl _GlobeView) over the intelligence API — grids, generators, flows, vessels, observability gaps", + "type": "module", + "engines": { + "node": ">=20" + }, + "scripts": { + "dev": "vite", + "build": "tsc -p tsconfig.json --noEmit && vite build", + "preview": "vite preview" + }, + "dependencies": { + "deck.gl": "^9.0.38" + }, + "devDependencies": { + "typescript": "^5.7.2", + "vite": "^6.0.5" + } +} diff --git a/intelligence/packages/globe/src/main.ts b/intelligence/packages/globe/src/main.ts new file mode 100644 index 0000000..5493888 --- /dev/null +++ b/intelligence/packages/globe/src/main.ts @@ -0,0 +1,2 @@ +// @pact-tailor/globe — deck.gl globe entry point. Populated in the globe commit. +export {}; diff --git a/intelligence/packages/globe/tsconfig.json b/intelligence/packages/globe/tsconfig.json new file mode 100644 index 0000000..d661a1c --- /dev/null +++ b/intelligence/packages/globe/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "noEmit": true, + "isolatedModules": true + }, + "include": ["src"] +} diff --git a/intelligence/packages/graph/package.json b/intelligence/packages/graph/package.json new file mode 100644 index 0000000..58af249 --- /dev/null +++ b/intelligence/packages/graph/package.json @@ -0,0 +1,21 @@ +{ + "name": "@pact-tailor/graph", + "version": "0.1.0", + "private": true, + "description": "In-process bitemporal property graph — typed nodes/edges, upstream/downstream traversal, supply-chain path queries", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": "./dist/index.js" + }, + "engines": { + "node": ">=20" + }, + "scripts": { + "build": "tsc -p tsconfig.json" + }, + "dependencies": { + "@pact-tailor/ontology": "*" + } +} diff --git a/intelligence/packages/graph/src/index.ts b/intelligence/packages/graph/src/index.ts new file mode 100644 index 0000000..91ee4fb --- /dev/null +++ b/intelligence/packages/graph/src/index.ts @@ -0,0 +1,2 @@ +// @pact-tailor/graph — bitemporal property graph. Populated in the graph commit. +export {}; diff --git a/intelligence/packages/graph/tsconfig.json b/intelligence/packages/graph/tsconfig.json new file mode 100644 index 0000000..5a24989 --- /dev/null +++ b/intelligence/packages/graph/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/intelligence/packages/intel/package.json b/intelligence/packages/intel/package.json new file mode 100644 index 0000000..9eede4a --- /dev/null +++ b/intelligence/packages/intel/package.json @@ -0,0 +1,23 @@ +{ + "name": "@pact-tailor/intel", + "version": "0.1.0", + "private": true, + "description": "Rule-based derived intelligence — facility utilisation estimation, cargo inference, anomaly detection, gap synthesis. Every output is an evidence-linked Inference record", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": "./dist/index.js" + }, + "engines": { + "node": ">=20" + }, + "scripts": { + "build": "tsc -p tsconfig.json" + }, + "dependencies": { + "@pact-tailor/graph": "*", + "@pact-tailor/ontology": "*", + "@pact-tailor/state": "*" + } +} diff --git a/intelligence/packages/intel/src/index.ts b/intelligence/packages/intel/src/index.ts new file mode 100644 index 0000000..5cd3b3d --- /dev/null +++ b/intelligence/packages/intel/src/index.ts @@ -0,0 +1,2 @@ +// @pact-tailor/intel — derived intelligence rules. Populated in the intel commit. +export {}; diff --git a/intelligence/packages/intel/tsconfig.json b/intelligence/packages/intel/tsconfig.json new file mode 100644 index 0000000..5a24989 --- /dev/null +++ b/intelligence/packages/intel/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/intelligence/packages/ontology/package.json b/intelligence/packages/ontology/package.json new file mode 100644 index 0000000..f48ed12 --- /dev/null +++ b/intelligence/packages/ontology/package.json @@ -0,0 +1,22 @@ +{ + "name": "@pact-tailor/ontology", + "version": "0.1.0", + "private": true, + "description": "Canonical entity/relationship/observation/inference envelopes: TS types, ajv validators, ID grammar", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": "./dist/index.js" + }, + "engines": { + "node": ">=20" + }, + "scripts": { + "build": "tsc -p tsconfig.json" + }, + "dependencies": { + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1" + } +} diff --git a/intelligence/packages/ontology/src/index.ts b/intelligence/packages/ontology/src/index.ts new file mode 100644 index 0000000..6bd32f2 --- /dev/null +++ b/intelligence/packages/ontology/src/index.ts @@ -0,0 +1,3 @@ +// @pact-tailor/ontology — canonical envelopes, validators, and ID grammar. +// Populated in the ontology commit; this placeholder keeps the workspace building. +export {}; diff --git a/intelligence/packages/ontology/tsconfig.json b/intelligence/packages/ontology/tsconfig.json new file mode 100644 index 0000000..5a24989 --- /dev/null +++ b/intelligence/packages/ontology/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/intelligence/packages/pact-bridge/package.json b/intelligence/packages/pact-bridge/package.json new file mode 100644 index 0000000..cd00849 --- /dev/null +++ b/intelligence/packages/pact-bridge/package.json @@ -0,0 +1,22 @@ +{ + "name": "@pact-tailor/pact-bridge", + "version": "0.1.0", + "private": true, + "description": "Bridges derived intelligence into PACT — inference records become `fact`-type proposals; competing hypotheses resolve via propose/object/consensus", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": "./dist/index.js" + }, + "engines": { + "node": ">=20" + }, + "scripts": { + "build": "tsc -p tsconfig.json" + }, + "dependencies": { + "@pact-tailor/intel": "*", + "@pact-tailor/ontology": "*" + } +} diff --git a/intelligence/packages/pact-bridge/src/index.ts b/intelligence/packages/pact-bridge/src/index.ts new file mode 100644 index 0000000..c26c923 --- /dev/null +++ b/intelligence/packages/pact-bridge/src/index.ts @@ -0,0 +1,2 @@ +// @pact-tailor/pact-bridge — inference → PACT fact proposals. Populated in the pact-bridge commit. +export {}; diff --git a/intelligence/packages/pact-bridge/tsconfig.json b/intelligence/packages/pact-bridge/tsconfig.json new file mode 100644 index 0000000..5a24989 --- /dev/null +++ b/intelligence/packages/pact-bridge/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/intelligence/packages/registry/package.json b/intelligence/packages/registry/package.json new file mode 100644 index 0000000..2227fa8 --- /dev/null +++ b/intelligence/packages/registry/package.json @@ -0,0 +1,22 @@ +{ + "name": "@pact-tailor/registry", + "version": "0.1.0", + "private": true, + "description": "YAML registry loaders (grids, sources, gaps, structural seed), cross-reference validation, entity-resolution candidates", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": "./dist/index.js" + }, + "engines": { + "node": ">=20" + }, + "scripts": { + "build": "tsc -p tsconfig.json" + }, + "dependencies": { + "@pact-tailor/ontology": "*", + "yaml": "^2.6.1" + } +} diff --git a/intelligence/packages/registry/src/index.ts b/intelligence/packages/registry/src/index.ts new file mode 100644 index 0000000..fb2cb6e --- /dev/null +++ b/intelligence/packages/registry/src/index.ts @@ -0,0 +1,2 @@ +// @pact-tailor/registry — registry loading and validation. Populated in the registry commit. +export {}; diff --git a/intelligence/packages/registry/tsconfig.json b/intelligence/packages/registry/tsconfig.json new file mode 100644 index 0000000..5a24989 --- /dev/null +++ b/intelligence/packages/registry/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/intelligence/packages/state/package.json b/intelligence/packages/state/package.json new file mode 100644 index 0000000..1cdef75 --- /dev/null +++ b/intelligence/packages/state/package.json @@ -0,0 +1,22 @@ +{ + "name": "@pact-tailor/state", + "version": "0.1.0", + "private": true, + "description": "Real-time state engine — observation store, current-state projections (event-time semantics, corrections), window aggregation, delta bus", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": "./dist/index.js" + }, + "engines": { + "node": ">=20" + }, + "scripts": { + "build": "tsc -p tsconfig.json" + }, + "dependencies": { + "@pact-tailor/ontology": "*", + "@pact-tailor/store": "*" + } +} diff --git a/intelligence/packages/state/src/index.ts b/intelligence/packages/state/src/index.ts new file mode 100644 index 0000000..0dbc94b --- /dev/null +++ b/intelligence/packages/state/src/index.ts @@ -0,0 +1,2 @@ +// @pact-tailor/state — real-time state engine. Populated in the state commit. +export {}; diff --git a/intelligence/packages/state/tsconfig.json b/intelligence/packages/state/tsconfig.json new file mode 100644 index 0000000..5a24989 --- /dev/null +++ b/intelligence/packages/state/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/intelligence/packages/store/package.json b/intelligence/packages/store/package.json new file mode 100644 index 0000000..4dc96d8 --- /dev/null +++ b/intelligence/packages/store/package.json @@ -0,0 +1,21 @@ +{ + "name": "@pact-tailor/store", + "version": "0.1.0", + "private": true, + "description": "Embedded bronze/silver/gold medallion storage — verbatim raw artifacts, append-only observations, hash-chained gold deltas", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": "./dist/index.js" + }, + "engines": { + "node": ">=20" + }, + "scripts": { + "build": "tsc -p tsconfig.json" + }, + "dependencies": { + "@pact-tailor/ontology": "*" + } +} diff --git a/intelligence/packages/store/src/index.ts b/intelligence/packages/store/src/index.ts new file mode 100644 index 0000000..c0d2c2a --- /dev/null +++ b/intelligence/packages/store/src/index.ts @@ -0,0 +1,2 @@ +// @pact-tailor/store — medallion storage. Populated in the store commit. +export {}; diff --git a/intelligence/packages/store/tsconfig.json b/intelligence/packages/store/tsconfig.json new file mode 100644 index 0000000..5a24989 --- /dev/null +++ b/intelligence/packages/store/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/intelligence/tsconfig.base.json b/intelligence/tsconfig.base.json new file mode 100644 index 0000000..9021ef7 --- /dev/null +++ b/intelligence/tsconfig.base.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "Node16", + "moduleResolution": "Node16", + "lib": ["ES2022"], + "strict": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "noUncheckedIndexedAccess": true + } +} From 7e51f72d002402d84b4cc4092c5bf7b4a0e2b68e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 01:37:26 +0000 Subject: [PATCH 02/16] =?UTF-8?q?feat(intelligence):=20ontology=20?= =?UTF-8?q?=E2=80=94=20canonical=20envelopes,=20validators,=20ID=20grammar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JSON Schema 2020-12 set for all nine envelopes (entity, relationship, observation, source, observability-gap, inference, grid, telemetry-feed, sdui-panel) with per-type property sub-schemas; the three data classes are structurally separate envelopes so derived intelligence cannot masquerade as telemetry. TS mirror types, Ajv validators, slug ID grammar, zero-dep ULID. 58 tests. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014yVGqBtvM1zNAd9tvUVNzm --- intelligence/packages/ontology/package.json | 3 +- intelligence/packages/ontology/src/ids.ts | 122 ++++ intelligence/packages/ontology/src/index.ts | 8 +- intelligence/packages/ontology/src/metrics.ts | 107 ++++ intelligence/packages/ontology/src/types.ts | 589 ++++++++++++++++++ intelligence/packages/ontology/src/ulid.ts | 76 +++ .../packages/ontology/src/validate.ts | 134 ++++ .../packages/ontology/test/ids.test.mjs | 84 +++ .../packages/ontology/test/ulid.test.mjs | 50 ++ .../packages/ontology/test/validate.test.mjs | 492 +++++++++++++++ intelligence/schemas/defs.schema.json | 119 ++++ .../entity-types/generator.properties.json | 19 + .../schemas/entity-types/grid.properties.json | 11 + .../interconnector.properties.json | 14 + .../schemas/entity-types/mine.properties.json | 18 + .../entity-types/pipeline.properties.json | 14 + .../schemas/entity-types/port.properties.json | 11 + .../entity-types/smelter.properties.json | 13 + .../entity-types/terminal.properties.json | 14 + .../entity-types/vessel.properties.json | 18 + intelligence/schemas/entity.schema.json | 129 ++++ intelligence/schemas/geojson.schema.json | 79 +++ intelligence/schemas/grid.schema.json | 120 ++++ intelligence/schemas/inference.schema.json | 103 +++ .../schemas/observability-gap.schema.json | 91 +++ intelligence/schemas/observation.schema.json | 35 ++ intelligence/schemas/relationship.schema.json | 75 +++ intelligence/schemas/sdui-panel.schema.json | 57 ++ intelligence/schemas/source.schema.json | 71 +++ .../schemas/telemetry-feed.schema.json | 23 + 30 files changed, 2696 insertions(+), 3 deletions(-) create mode 100644 intelligence/packages/ontology/src/ids.ts create mode 100644 intelligence/packages/ontology/src/metrics.ts create mode 100644 intelligence/packages/ontology/src/types.ts create mode 100644 intelligence/packages/ontology/src/ulid.ts create mode 100644 intelligence/packages/ontology/src/validate.ts create mode 100644 intelligence/packages/ontology/test/ids.test.mjs create mode 100644 intelligence/packages/ontology/test/ulid.test.mjs create mode 100644 intelligence/packages/ontology/test/validate.test.mjs create mode 100644 intelligence/schemas/defs.schema.json create mode 100644 intelligence/schemas/entity-types/generator.properties.json create mode 100644 intelligence/schemas/entity-types/grid.properties.json create mode 100644 intelligence/schemas/entity-types/interconnector.properties.json create mode 100644 intelligence/schemas/entity-types/mine.properties.json create mode 100644 intelligence/schemas/entity-types/pipeline.properties.json create mode 100644 intelligence/schemas/entity-types/port.properties.json create mode 100644 intelligence/schemas/entity-types/smelter.properties.json create mode 100644 intelligence/schemas/entity-types/terminal.properties.json create mode 100644 intelligence/schemas/entity-types/vessel.properties.json create mode 100644 intelligence/schemas/entity.schema.json create mode 100644 intelligence/schemas/geojson.schema.json create mode 100644 intelligence/schemas/grid.schema.json create mode 100644 intelligence/schemas/inference.schema.json create mode 100644 intelligence/schemas/observability-gap.schema.json create mode 100644 intelligence/schemas/observation.schema.json create mode 100644 intelligence/schemas/relationship.schema.json create mode 100644 intelligence/schemas/sdui-panel.schema.json create mode 100644 intelligence/schemas/source.schema.json create mode 100644 intelligence/schemas/telemetry-feed.schema.json diff --git a/intelligence/packages/ontology/package.json b/intelligence/packages/ontology/package.json index f48ed12..f6cc0f2 100644 --- a/intelligence/packages/ontology/package.json +++ b/intelligence/packages/ontology/package.json @@ -13,7 +13,8 @@ "node": ">=20" }, "scripts": { - "build": "tsc -p tsconfig.json" + "build": "tsc -p tsconfig.json", + "test": "npm run build && node --test \"test/*.test.mjs\"" }, "dependencies": { "ajv": "^8.17.1", diff --git a/intelligence/packages/ontology/src/ids.ts b/intelligence/packages/ontology/src/ids.ts new file mode 100644 index 0000000..9981ce7 --- /dev/null +++ b/intelligence/packages/ontology/src/ids.ts @@ -0,0 +1,122 @@ +// ID grammar for the PACT.TAILOR intelligence workspace. +// +// Entity IDs: typeCode:namespace[:slug] e.g. gen:au-nem:bayswater +// Relationships: rel: +// Sources: source:: +// Gaps: gap:: +// Feeds: feed::[:] +// Inferences: infer:YYYY-MM-DD: + +import type { EntityType } from "./types.js"; + +export const ENTITY_ID_RE = /^[a-z][a-z0-9]*:[a-z0-9][a-z0-9-]*(:[a-z0-9][a-z0-9-]*)?$/; +export const REL_ID_RE = /^rel:[a-z0-9][a-z0-9-]*$/; +export const SOURCE_ID_RE = /^source:[a-z0-9-]+:[a-z0-9-]+$/; +export const GAP_ID_RE = /^gap:[a-z0-9-]+:[a-z0-9-]+$/; +export const FEED_ID_RE = /^feed:[a-z0-9-]+:[a-z0-9-]+(:[a-z0-9-]+)?$/; +export const INFERENCE_ID_RE = /^infer:[0-9]{4}-[0-9]{2}-[0-9]{2}:[a-z0-9][a-z0-9-]*$/; + +export interface ParsedEntityId { + /** Leading type code, e.g. "gen" */ + typeCode: string; + /** Second segment, e.g. "au-nem" */ + namespace: string; + /** Optional third segment, or null when absent */ + slug: string | null; +} + +/** Parse an entity ID into its segments, or return null if malformed. */ +export function parseEntityId(id: string): ParsedEntityId | null { + if (!ENTITY_ID_RE.test(id)) return null; + const parts = id.split(":"); + const typeCode = parts[0]; + const namespace = parts[1]; + if (typeCode === undefined || namespace === undefined) return null; + return { typeCode, namespace, slug: parts[2] ?? null }; +} + +export function isEntityId(id: string): boolean { + return ENTITY_ID_RE.test(id); +} + +export function isRelId(id: string): boolean { + return REL_ID_RE.test(id); +} + +export function isSourceId(id: string): boolean { + return SOURCE_ID_RE.test(id); +} + +export function isGapId(id: string): boolean { + return GAP_ID_RE.test(id); +} + +export function isFeedId(id: string): boolean { + return FEED_ID_RE.test(id); +} + +export function isInferenceId(id: string): boolean { + return INFERENCE_ID_RE.test(id); +} + +/** + * Expected entity-ID type code(s) per entity_type. + * "mine" is shared by CoalMine and Mine. + * (Observation has no code in the published grammar; "obs" is reserved for it.) + */ +export const TYPE_CODES: Readonly> = { + Grid: ["grid"], + GridRegion: ["region"], + Operator: ["operator"], + Generator: ["gen"], + StorageAsset: ["storage"], + TransmissionAsset: ["line"], + Interconnector: ["intercon"], + Substation: ["sub"], + CoalBasin: ["basin"], + CoalMine: ["mine"], + GasBasin: ["gasbasin"], + GasField: ["gasfield"], + GasProcessingPlant: ["gasplant"], + Pipeline: ["pipe"], + HydroReservoir: ["resv"], + IndustrialFacility: ["facility"], + Smelter: ["smelter"], + SteelMill: ["steel"], + Mine: ["mine"], + Refinery: ["refinery"], + LNGPlant: ["lng"], + ChemicalPlant: ["chem"], + DataCentre: ["dc"], + Port: ["port"], + Terminal: ["terminal"], + Berth: ["berth"], + Railway: ["rail"], + Train: ["train"], + Vessel: ["vessel"], + Company: ["company"], + Commodity: ["commodity"], + Product: ["product"], + Market: ["market"], + Destination: ["dest"], + Sensor: ["sensor"], + TelemetryFeed: ["feed"], + Observation: ["obs"], + Source: ["source"], + Evidence: ["evidence"], + Inference: ["infer"], + ObservabilityGap: ["gap"], +}; + +/** + * Check that an entity_id's leading type code matches the expected code(s) + * for the given entity_type. Shared codes (e.g. "mine") are allowed. + * Returns false for malformed IDs or unknown entity types. + */ +export function checkIdMatchesType(entityId: string, entityType: EntityType): boolean { + const parsed = parseEntityId(entityId); + if (parsed === null) return false; + const codes = TYPE_CODES[entityType]; + if (codes === undefined) return false; + return codes.includes(parsed.typeCode); +} diff --git a/intelligence/packages/ontology/src/index.ts b/intelligence/packages/ontology/src/index.ts index 6bd32f2..a728b29 100644 --- a/intelligence/packages/ontology/src/index.ts +++ b/intelligence/packages/ontology/src/index.ts @@ -1,3 +1,7 @@ // @pact-tailor/ontology — canonical envelopes, validators, and ID grammar. -// Populated in the ontology commit; this placeholder keeps the workspace building. -export {}; + +export * from "./types.js"; +export * from "./ids.js"; +export * from "./ulid.js"; +export * from "./metrics.js"; +export * from "./validate.js"; diff --git a/intelligence/packages/ontology/src/metrics.ts b/intelligence/packages/ontology/src/metrics.ts new file mode 100644 index 0000000..80e3640 --- /dev/null +++ b/intelligence/packages/ontology/src/metrics.ts @@ -0,0 +1,107 @@ +// Canonical metric vocabulary. Observations may carry unknown metrics +// (the schema only enforces the dotted-lowercase grammar); unknown metrics +// should be flagged by tooling via isKnownMetric(). + +export interface MetricSpec { + /** Canonical unit, or "object" for structured values, or null for dimensionless. */ + unit: string | null; + description: string; +} + +export const METRICS = { + "power.output.mw": { + unit: "MW", + description: "Instantaneous electrical output of a generator/unit", + }, + "power.load.mw": { + unit: "MW", + description: "Metered electrical load of a facility", + }, + "power.load.estimated_mw": { + unit: "MW", + description: "Estimated (not metered) electrical load of a facility", + }, + "power.capacity_factor": { + unit: null, + description: "Output / nameplate capacity, dimensionless 0..1", + }, + "market.price.energy": { + unit: "AUD/MWh", + description: "Spot/dispatch energy price", + }, + "grid.demand.mw": { + unit: "MW", + description: "Total grid or region demand", + }, + "grid.frequency.hz": { + unit: "Hz", + description: "System frequency", + }, + "grid.generation.mw": { + unit: "MW", + description: "Total grid or region generation", + }, + "grid.generation.fuel_mix": { + unit: "object", + description: "Generation by fuel, object keyed by fuel with MW values", + }, + "intercon.flow.mw": { + unit: "MW", + description: "Interconnector flow, signed (positive = forward direction)", + }, + "storage.level.pct": { + unit: "%", + description: "Storage level as percentage of usable capacity", + }, + "storage.flow.mw": { + unit: "MW", + description: "Storage charge/discharge rate, signed", + }, + "vessel.position": { + unit: "object", + description: "Vessel position object {lat, lon, speed_kn, heading_deg, draught_m?}", + }, + "vessel.speed.kn": { + unit: "kn", + description: "Vessel speed over ground", + }, + "vessel.heading.deg": { + unit: "deg", + description: "Vessel heading, degrees true", + }, + "vessel.draught.m": { + unit: "m", + description: "Vessel reported draught", + }, + "weather.temp.c": { + unit: "degC", + description: "Air temperature", + }, + "weather.wind.speed_ms": { + unit: "m/s", + description: "Wind speed", + }, + "gas.pipeline.flow.tj_day": { + unit: "TJ/day", + description: "Gas pipeline flow rate", + }, + "production.rate.tph": { + unit: "t/h", + description: "Industrial production rate, tonnes per hour", + }, +} as const satisfies Record; + +export type KnownMetric = keyof typeof METRICS; + +/** Grammar every metric name must satisfy (known or not). */ +export const METRIC_RE = /^[a-z][a-z0-9_]*(\.[a-z0-9_]+)+$/; + +/** True if the metric is part of the canonical vocabulary. */ +export function isKnownMetric(metric: string): metric is KnownMetric { + return Object.prototype.hasOwnProperty.call(METRICS, metric); +} + +/** True if the metric name satisfies the metric grammar (canonical or not). */ +export function isValidMetricName(metric: string): boolean { + return METRIC_RE.test(metric); +} diff --git a/intelligence/packages/ontology/src/types.ts b/intelligence/packages/ontology/src/types.ts new file mode 100644 index 0000000..03347a2 --- /dev/null +++ b/intelligence/packages/ontology/src/types.ts @@ -0,0 +1,589 @@ +// Canonical TypeScript mirrors of the JSON Schemas in intelligence/schemas/. +// Keep these in lock-step with the schemas — the schemas are the source of truth. + +// --------------------------------------------------------------------------- +// GeoJSON (minimal WGS84 subset — mirrors geojson.schema.json) +// --------------------------------------------------------------------------- + +/** [lon, lat] or [lon, lat, elevation] */ +export type Position = [number, number] | [number, number, number] | number[]; + +export interface GeoJSONPoint { + type: "Point"; + coordinates: Position; +} + +export interface GeoJSONLineString { + type: "LineString"; + coordinates: Position[]; +} + +export interface GeoJSONPolygon { + type: "Polygon"; + coordinates: Position[][]; +} + +export interface GeoJSONMultiPolygon { + type: "MultiPolygon"; + coordinates: Position[][][]; +} + +export type GeoJSONGeometry = + | GeoJSONPoint + | GeoJSONLineString + | GeoJSONPolygon + | GeoJSONMultiPolygon; + +// --------------------------------------------------------------------------- +// Entity (entity.schema.json) +// --------------------------------------------------------------------------- + +export type EntityType = + | "Grid" + | "GridRegion" + | "Operator" + | "Generator" + | "StorageAsset" + | "TransmissionAsset" + | "Interconnector" + | "Substation" + | "CoalBasin" + | "CoalMine" + | "GasBasin" + | "GasField" + | "GasProcessingPlant" + | "Pipeline" + | "HydroReservoir" + | "IndustrialFacility" + | "Smelter" + | "SteelMill" + | "Mine" + | "Refinery" + | "LNGPlant" + | "ChemicalPlant" + | "DataCentre" + | "Port" + | "Terminal" + | "Berth" + | "Railway" + | "Train" + | "Vessel" + | "Company" + | "Commodity" + | "Product" + | "Market" + | "Destination" + | "Sensor" + | "TelemetryFeed" + | "Observation" + | "Source" + | "Evidence" + | "Inference" + | "ObservabilityGap"; + +export type Observability = + | "KNOWN_LIVE" + | "ESTIMATED" + | "NOT_OBSERVABLE" + | "INSTRUMENTATION_OPPORTUNITY" + | "UNCLASSIFIED"; + +export interface ExternalIds { + aemo_duid?: string[]; + aemo_station_id?: string; + aemo_region_id?: string; + entsoe_eic?: string; + eia_plant_code?: string; + gem_id?: string; + imo?: string; + mmsi?: string; + callsign?: string; + unlocode?: string; + abn?: string; + lei?: string; + wikidata_qid?: string; + osm_id?: string; +} + +// Typed per-entity-type property bags (entity-types/*.properties.json). +// All are open (additionalProperties: true) — extra keys are allowed. + +export type GeneratorFuel = + | "coal" + | "gas" + | "hydro" + | "wind" + | "solar" + | "battery" + | "pumped-hydro" + | "diesel" + | "biomass" + | "nuclear" + | "other"; + +export type GeneratorStatus = + | "operating" + | "mothballed" + | "retired" + | "announced" + | "construction"; + +export interface GeneratorProperties { + fuel?: GeneratorFuel; + capacity_mw?: number; + technology?: string; + commissioned?: number; + unit_count?: number; + status?: GeneratorStatus; + [key: string]: unknown; +} + +export interface InterconnectorProperties { + capacity_mw_forward?: number; + capacity_mw_reverse?: number; + from_region?: string; + to_region?: string; + hvdc?: boolean; + [key: string]: unknown; +} + +export interface VesselProperties { + vessel_type?: string; + dwt?: number; + loa_m?: number; + beam_m?: number; + flag?: string; + /** true marks exemplar/demo vessels that are NOT real ships */ + synthetic?: boolean; + [key: string]: unknown; +} + +export interface PortProperties { + harbour_type?: string; + max_draught_m?: number; + [key: string]: unknown; +} + +export interface TerminalProperties { + commodities?: string[]; + nameplate_mtpa?: number; + [key: string]: unknown; +} + +export interface SmelterProperties { + product?: string; + capacity_tpa?: number; + potlines?: number; + technology?: string; + [key: string]: unknown; +} + +export type MineType = "open-cut" | "underground" | "mixed"; + +export interface MineProperties { + commodities?: string[]; + mine_type?: MineType; + capacity_mtpa?: number; + status?: string; + [key: string]: unknown; +} + +export type PipelineCommodity = "gas" | "oil" | "water" | "slurry"; + +export interface PipelineProperties { + commodity?: PipelineCommodity; + length_km?: number; + capacity_tj_day?: number; + [key: string]: unknown; +} + +export interface GridProperties { + frequency_hz?: 50 | 60; + kind?: string; + [key: string]: unknown; +} + +export type EntityProperties = + | GeneratorProperties + | InterconnectorProperties + | VesselProperties + | PortProperties + | TerminalProperties + | SmelterProperties + | MineProperties + | PipelineProperties + | GridProperties + | Record; + +export interface EntityRecord { + entity_id: string; + entity_type: EntityType; + name: string; + aliases?: string[]; + data_class: "structural"; + observability: Observability; + geometry?: GeoJSONGeometry | null; + country?: string | null; + admin?: { region?: string; locality?: string }; + grid_id?: string | null; + properties?: EntityProperties; + external_ids?: ExternalIds; + sources: string[]; + valid_from?: string | null; + valid_to?: string | null; + notes?: string; + same_as?: string | null; +} + +// --------------------------------------------------------------------------- +// Relationship (relationship.schema.json) +// --------------------------------------------------------------------------- + +export type RelType = + | "PART_OF" + | "CONNECTED_TO" + | "INTERCONNECTED_WITH" + | "OWNED_BY" + | "OPERATED_BY" + | "EXTRACTS" + | "SUPPLIES" + | "MAY_SUPPLY" + | "TRANSPORTS" + | "GENERATES" + | "TRANSMITS" + | "IMPORTS_FROM" + | "EXPORTS_TO" + | "CONSUMES" + | "PRODUCES" + | "LOADS_AT" + | "DEPARTS_FROM" + | "ARRIVES_AT" + | "SHIPS_TO" + | "OBSERVED_BY" + | "REPORTED_BY" + | "INFERRED_FROM" + | "SUPPORTED_BY" + | "CONTRADICTED_BY"; + +export type RelMethod = "declared" | "derived" | "inferred" | "manual"; + +export type EvidenceKind = "observation" | "structural" | "inference"; + +/** Evidence item on a relationship ({kind, ref}). */ +export interface RelEvidenceItem { + kind: EvidenceKind; + ref: string; +} + +export interface RelationshipRecord { + rel_id: string; + rel_type: RelType; + from_id: string; + to_id: string; + confidence: number; + source: string; + method: RelMethod; + valid_from?: string | null; + valid_to?: string | null; + recorded_at?: string | null; + superseded_at?: string | null; + /** Required (minItems 1) when method === "inferred". */ + evidence?: RelEvidenceItem[]; + properties?: Record; + notes?: string; +} + +// --------------------------------------------------------------------------- +// Observation (observation.schema.json) +// --------------------------------------------------------------------------- + +export type Quality = "good" | "suspect" | "estimated" | "interpolated" | "missing"; + +/** Structured value for vessel.position observations. */ +export interface VesselPositionValue { + lat: number; + lon: number; + speed_kn?: number; + heading_deg?: number; + draught_m?: number; + [key: string]: unknown; +} + +export type ObservationValue = number | string | Record; + +export interface ObservationRecord { + observation_id: string; + entity_id: string; + metric: string; + value: ObservationValue; + event_time: string; + ingest_time: string; + source_id: string; + feed_id: string; + quality: Quality; + unit?: string; + is_correction?: boolean; + corrects?: string | null; + source_sequence?: string; + meta?: Record; +} + +// --------------------------------------------------------------------------- +// Source (source.schema.json) +// --------------------------------------------------------------------------- + +export type ApiType = "rest" | "websocket" | "file-drop" | "html" | "bulk-download" | "none"; +export type SourceDataClass = "live" | "static" | "both"; +export type LicensingClass = + | "open" + | "open-attribution" + | "registration-required" + | "commercial" + | "restricted"; +export type VerificationStatus = "verified_live" | "documented_only" | "unverified"; +export type Authentication = "none" | "api_key" | "oauth" | "registration"; + +export interface SourceRecord { + source_id: string; + publisher: string; + dataset: string; + api_type: ApiType; + url: string; + docs_url?: string; + coverage: { grids: string[]; countries: string[] }; + data_class: SourceDataClass; + entity_types?: string[]; + metrics?: string[]; + latency?: string; + resolution?: string; + update_frequency?: string; + history_depth?: string; + licensing: { class: LicensingClass; notes?: string }; + authentication?: Authentication; + cost?: string; + reliability?: number; + verification_status: VerificationStatus; + last_checked: string; + quality_notes?: string; + notes?: string; +} + +// --------------------------------------------------------------------------- +// Observability gap (observability-gap.schema.json) +// --------------------------------------------------------------------------- + +export type ValueBand = "low" | "medium" | "high" | "critical"; +export type GapPriority = "P1" | "P2" | "P3" | "P4"; +export type GapStatus = "open" | "proxied" | "instrumented" | "wont_do"; +export type CostBand = "low" | "medium" | "high"; + +export type InstrumentationKind = + | "ct_sensor" + | "revenue_meter" + | "power_quality_meter" + | "substation_gateway" + | "iot_gateway" + | "weather_station" + | "camera_ocr" + | "ais_receiver" + | "gnss" + | "scada_integration" + | "plc_integration" + | "opc_ua" + | "modbus" + | "mqtt_gateway" + | "environmental_sensor" + | "condition_sensor" + | "satellite_thermal" + | "satellite_optical" + | "partner_feed" + | "customer_api"; + +export interface InstrumentationOption { + kind: InstrumentationKind; + notes?: string; + indicative_cost_band?: CostBand; +} + +export interface ObservabilityGapRecord { + gap_id: string; + entity_id: string; + desired_metric: string; + commercial_value: ValueBand; + strategic_value: ValueBand; + priority: GapPriority; + status: GapStatus; + current_source?: string | null; + best_available_proxy?: { + source_id: string; + metric: string; + derivation?: string; + } | null; + required_resolution?: string; + required_accuracy?: string; + instrumentation_options?: InstrumentationOption[]; + estimated_complexity?: CostBand; + notes?: string; +} + +// --------------------------------------------------------------------------- +// Inference (inference.schema.json) +// --------------------------------------------------------------------------- + +export type InferenceTier = "A" | "B" | "C"; +export type InferenceStatus = + | "INFERRED" + | "CORROBORATED" + | "CONTESTED" + | "VERIFIED" + | "RETRACTED"; + +/** Evidence item on an inference — role discriminates supporting vs contradicting. */ +export interface EvidenceRef { + kind: EvidenceKind; + ref: string; + role: "supports" | "contradicts"; +} + +export interface SupportingEvidenceRef extends EvidenceRef { + role: "supports"; +} + +export interface ContraryEvidenceRef extends EvidenceRef { + role: "contradicts"; +} + +export interface InferenceRecord { + inference_id: string; + claim: string; + claim_structured: { + subject: string; + predicate: string; + object: string; + qualifiers?: Record; + }; + tier: InferenceTier; + confidence: number; + status: InferenceStatus; + /** rule:@, e.g. rule:load-estimate@1.2.0 */ + method: string; + evidence: SupportingEvidenceRef[]; + contrary_evidence?: ContraryEvidenceRef[]; + sources: string[]; + produced_at: string; + event_time_range?: { from?: string; to?: string }; + lineage?: { + correlation_id?: string; + in_response_to?: string | null; + prev_hash?: string | null; + }; + pact?: { fabric_id?: string; proposal_id?: string } | null; + notes?: string; +} + +// --------------------------------------------------------------------------- +// Grid registry record (grid.schema.json) +// --------------------------------------------------------------------------- + +export type GridKind = + | "synchronous_area" + | "interconnection" + | "iso_rto" + | "tso_area" + | "market" + | "balancing_authority" + | "national_grid" + | "regional_grid" + | "isolated_grid" + | "captive"; + +export type LiveDataAvailability = "rich" | "partial" | "minimal" | "none"; + +export type OperatorRole = + | "system_operator" + | "market_operator" + | "tso" + | "transmission_owner" + | "regulator"; + +export interface GridRecord { + grid_id: string; + name: string; + kind: GridKind; + countries: string[]; + frequency_hz: 50 | 60; + live_data: { + availability: LiveDataAvailability; + notes?: string; + source_ids?: string[]; + }; + sources: string[]; + parent_grid?: string | null; + operators?: Array<{ name: string; role: OperatorRole }>; + market?: { + structure?: string; + price_mechanism?: string; + dispatch_interval?: string; + } | null; + timezone?: string; + interconnections?: Array<{ + to_grid: string; + name?: string; + capacity_mw?: number | null; + }>; + demand_range_mw?: { min?: number; max?: number } | null; + notes?: string; +} + +// --------------------------------------------------------------------------- +// Telemetry feed (telemetry-feed.schema.json) +// --------------------------------------------------------------------------- + +export interface TelemetryFeedRecord { + feed_id: string; + source_id: string; + connector_id: string; + metrics: string[]; + cadence: string; + description?: string; + entity_scope?: string; + notes?: string; +} + +// --------------------------------------------------------------------------- +// SDUI panel document (sdui-panel.schema.json) +// --------------------------------------------------------------------------- + +export type SduiSituation = + | "no-live-data" + | "stale-telemetry" + | "has-gaps" + | "estimated-load" + | "anomaly-active" + | "normal"; + +export type SduiPanelKind = + | "headline-state" + | "timeseries" + | "gap-card" + | "graph-neighborhood" + | "inference-list" + | "vessel-track" + | "fuel-mix" + | "price-strip" + | "flow-arcs" + | "table" + | "markdown-note"; + +export interface SduiPanel { + panel: SduiPanelKind; + props?: Record; + data?: { endpoint: string }; +} + +export interface SduiPanelDoc { + entity_id: string; + entity_type: EntityType; + situation: SduiSituation[]; + layout: SduiPanel[]; +} diff --git a/intelligence/packages/ontology/src/ulid.ts b/intelligence/packages/ontology/src/ulid.ts new file mode 100644 index 0000000..d4b6e52 --- /dev/null +++ b/intelligence/packages/ontology/src/ulid.ts @@ -0,0 +1,76 @@ +// Zero-dependency ULID (https://github.com/ulid/spec) built on node:crypto. +// Monotonicity within a millisecond is NOT guaranteed (not required here). + +import { randomBytes } from "node:crypto"; + +/** Crockford base32 alphabet (no I, L, O, U). */ +const ENCODING = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"; +const TIME_LEN = 10; +const RANDOM_LEN = 16; +const MAX_TIME = 2 ** 48 - 1; + +export const ULID_RE = /^[0-7][0-9A-HJKMNP-TV-Z]{25}$/; + +const DECODE: Record = {}; +for (let i = 0; i < ENCODING.length; i++) { + const ch = ENCODING[i]; + if (ch !== undefined) DECODE[ch] = i; +} + +function encodeTime(timeMs: number): string { + if (!Number.isInteger(timeMs) || timeMs < 0 || timeMs > MAX_TIME) { + throw new RangeError(`ulid: time must be an integer in [0, 2^48), got ${timeMs}`); + } + let t = timeMs; + const chars = new Array(TIME_LEN); + for (let i = TIME_LEN - 1; i >= 0; i--) { + chars[i] = ENCODING[t % 32] as string; + t = Math.floor(t / 32); + } + return chars.join(""); +} + +function encodeRandom(): string { + // 80 bits of randomness -> 16 base32 chars (5 bits each). + const bytes = randomBytes(10); + let out = ""; + let buffer = 0; + let bitsInBuffer = 0; + for (const byte of bytes) { + buffer = (buffer << 8) | byte; + bitsInBuffer += 8; + while (bitsInBuffer >= 5) { + bitsInBuffer -= 5; + out += ENCODING[(buffer >>> bitsInBuffer) & 0x1f]; + buffer &= (1 << bitsInBuffer) - 1; + } + } + return out; +} + +/** Generate a ULID for the given timestamp (defaults to now). */ +export function ulid(timeMs: number = Date.now()): string { + return encodeTime(timeMs) + encodeRandom(); +} + +/** Decode the millisecond timestamp encoded in a ULID. Throws on malformed input. */ +export function ulidTime(id: string): number { + if (typeof id !== "string" || id.length !== TIME_LEN + RANDOM_LEN || !ULID_RE.test(id)) { + throw new TypeError(`ulid: malformed ULID "${id}"`); + } + let t = 0; + for (let i = 0; i < TIME_LEN; i++) { + const ch = id[i] as string; + const v = DECODE[ch]; + if (v === undefined) { + throw new TypeError(`ulid: invalid character "${ch}" in "${id}"`); + } + t = t * 32 + v; + } + return t; +} + +/** True if the string is a well-formed ULID. */ +export function isUlid(id: string): boolean { + return ULID_RE.test(id); +} diff --git a/intelligence/packages/ontology/src/validate.ts b/intelligence/packages/ontology/src/validate.ts new file mode 100644 index 0000000..c7797cd --- /dev/null +++ b/intelligence/packages/ontology/src/validate.ts @@ -0,0 +1,134 @@ +// Runtime schema validation: loads the JSON Schemas from intelligence/schemas/ +// into a single Ajv 2020-12 instance and exposes typed validators per envelope. + +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import type { AnySchemaObject, ErrorObject, ValidateFunction } from "ajv/dist/2020.js"; +// Node16 ESM<->CJS interop: the callable plugin lives on .default at runtime +// (ajv-formats sets `exports.default = formatsPlugin`) and in the CJS typings. +import addFormatsModule from "ajv-formats"; +const addFormats = addFormatsModule.default; +import type { + EntityRecord, + GridRecord, + InferenceRecord, + ObservabilityGapRecord, + ObservationRecord, + RelationshipRecord, + SduiPanelDoc, + SourceRecord, + TelemetryFeedRecord, +} from "./types.js"; + +export const SCHEMA_BASE = "https://pact.tailor.au/intelligence/schemas/"; + +/** + * Default location of the schema JSON files. + * Compiled output lives at intelligence/packages/ontology/dist/validate.js, + * so ../../../schemas resolves to intelligence/schemas. + */ +export function defaultSchemasDir(): string { + const here = path.dirname(fileURLToPath(import.meta.url)); + return path.resolve(here, "../../../schemas"); +} + +export type ValidationResult = + | { ok: true; value: T } + | { ok: false; errors: string[] }; + +export type Validate = (input: unknown) => ValidationResult; + +export interface Validators { + entity: Validate; + relationship: Validate; + observation: Validate; + source: Validate; + gap: Validate; + inference: Validate; + grid: Validate; + feed: Validate; + sduiPanel: Validate; + /** The underlying Ajv instance (all schemas registered by $id). */ + ajv: Ajv2020; +} + +function loadSchemaFiles(dir: string): AnySchemaObject[] { + const schemas: AnySchemaObject[] = []; + for (const file of readdirSync(dir)) { + if (file.endsWith(".schema.json")) { + schemas.push(JSON.parse(readFileSync(path.join(dir, file), "utf8")) as AnySchemaObject); + } + } + const subDir = path.join(dir, "entity-types"); + for (const file of readdirSync(subDir)) { + if (file.endsWith(".json")) { + schemas.push(JSON.parse(readFileSync(path.join(subDir, file), "utf8")) as AnySchemaObject); + } + } + return schemas; +} + +function formatErrors(errors: ErrorObject[]): string[] { + return errors.map((e) => { + const where = e.instancePath === "" ? "(root)" : e.instancePath; + const params = e.params as Record; + let detail = ""; + if (e.keyword === "additionalProperties" && typeof params["additionalProperty"] === "string") { + detail = ` ("${String(params["additionalProperty"])}")`; + } else if (e.keyword === "enum" && params["allowedValues"] !== undefined) { + detail = `: ${JSON.stringify(params["allowedValues"])}`; + } + return `${where} ${e.message ?? "is invalid"}${detail} [${e.schemaPath}]`; + }); +} + +/** + * Load every schema from the schemas directory, compile them in one + * Ajv 2020-12 instance, and return typed per-envelope validators. + * Throws if any schema fails to compile or has unresolved $refs. + */ +export function createValidators(schemasDir: string = defaultSchemasDir()): Validators { + const ajv = new Ajv2020({ + allErrors: true, + strict: true, + allowUnionTypes: true, + }); + addFormats(ajv); + + for (const schema of loadSchemaFiles(schemasDir)) { + ajv.addSchema(schema); + } + + const compiled = (name: string): ValidateFunction => { + // getSchema compiles on first access; unresolved $refs throw here. + const fn = ajv.getSchema(`${SCHEMA_BASE}${name}`); + if (fn === undefined) { + throw new Error(`ontology: schema not registered: ${name}`); + } + return fn; + }; + + const wrap = (fn: ValidateFunction): Validate => { + return (input: unknown): ValidationResult => { + if (fn(input)) { + return { ok: true, value: input as T }; + } + return { ok: false, errors: formatErrors(fn.errors ?? []) }; + }; + }; + + return { + entity: wrap(compiled("entity.schema.json")), + relationship: wrap(compiled("relationship.schema.json")), + observation: wrap(compiled("observation.schema.json")), + source: wrap(compiled("source.schema.json")), + gap: wrap(compiled("observability-gap.schema.json")), + inference: wrap(compiled("inference.schema.json")), + grid: wrap(compiled("grid.schema.json")), + feed: wrap(compiled("telemetry-feed.schema.json")), + sduiPanel: wrap(compiled("sdui-panel.schema.json")), + ajv, + }; +} diff --git a/intelligence/packages/ontology/test/ids.test.mjs b/intelligence/packages/ontology/test/ids.test.mjs new file mode 100644 index 0000000..6f87fca --- /dev/null +++ b/intelligence/packages/ontology/test/ids.test.mjs @@ -0,0 +1,84 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + parseEntityId, + isEntityId, + isRelId, + isSourceId, + isGapId, + isFeedId, + isInferenceId, + checkIdMatchesType, + TYPE_CODES, +} from "../dist/index.js"; + +test("parseEntityId: three-segment id", () => { + assert.deepEqual(parseEntityId("gen:au-nem:bayswater"), { + typeCode: "gen", + namespace: "au-nem", + slug: "bayswater", + }); +}); + +test("parseEntityId: two-segment id has null slug", () => { + assert.deepEqual(parseEntityId("grid:au-nem"), { + typeCode: "grid", + namespace: "au-nem", + slug: null, + }); +}); + +test("parseEntityId: malformed ids return null", () => { + for (const bad of [ + "", + "gen", + "Gen:au-nem:x", + "gen:au-nem:Bayswater", + "gen:-au:x", + "gen:au:x:y", + "gen:au_nem:x", + "1gen:au:x", + ":au:x", + ]) { + assert.equal(parseEntityId(bad), null, `expected null for ${JSON.stringify(bad)}`); + } +}); + +test("id predicate accept/reject table", () => { + const table = [ + // [fn, accepted[], rejected[]] + [isEntityId, ["gen:au-nem:bayswater", "grid:au-nem", "vessel:imo:9700100"], ["gen", "GEN:au:x", "gen:au:x:y", "rel bad"]], + [isRelId, ["rel:bayswater-owned-by-agl", "rel:x1"], ["rel:", "rel:X", "gen:au-nem:x", "rel:a:b"]], + [isSourceId, ["source:aemo:nem-dispatch", "source:gem:coal-plants"], ["source:aemo", "source:aemo:nem:extra", "src:aemo:x", "source:AEMO:x"]], + [isGapId, ["gap:au-nem:tomago-load"], ["gap:au-nem", "gap:x:y:z", "gaps:a:b"]], + [isFeedId, ["feed:aemo:scada", "feed:aemo:scada:nsw1"], ["feed:aemo", "feed:aemo:scada:nsw1:extra", "feeds:a:b"]], + [isInferenceId, ["infer:2026-08-12:tomago-restart"], ["infer:2026-8-12:x", "infer:tomago", "infer:2026-08-12:", "infer:2026-08-12:X"]], + ]; + for (const [fn, accepted, rejected] of table) { + for (const id of accepted) { + assert.equal(fn(id), true, `${fn.name} should accept ${JSON.stringify(id)}`); + } + for (const id of rejected) { + assert.equal(fn(id), false, `${fn.name} should reject ${JSON.stringify(id)}`); + } + } +}); + +test("TYPE_CODES covers all 41 entity types", () => { + assert.equal(Object.keys(TYPE_CODES).length, 41); + assert.deepEqual(TYPE_CODES.Generator, ["gen"]); + assert.deepEqual(TYPE_CODES.CoalMine, ["mine"]); + assert.deepEqual(TYPE_CODES.Mine, ["mine"]); +}); + +test("checkIdMatchesType", () => { + assert.equal(checkIdMatchesType("gen:au-nem:bayswater", "Generator"), true); + assert.equal(checkIdMatchesType("grid:au-nem", "Grid"), true); + // shared code: mine works for both CoalMine and Mine + assert.equal(checkIdMatchesType("mine:au-nsw:mount-thorley", "CoalMine"), true); + assert.equal(checkIdMatchesType("mine:au-wa:mount-whaleback", "Mine"), true); + // mismatched code + assert.equal(checkIdMatchesType("gen:au-nem:bayswater", "Vessel"), false); + // malformed id + assert.equal(checkIdMatchesType("not an id", "Generator"), false); +}); diff --git a/intelligence/packages/ontology/test/ulid.test.mjs b/intelligence/packages/ontology/test/ulid.test.mjs new file mode 100644 index 0000000..b8ddc19 --- /dev/null +++ b/intelligence/packages/ontology/test/ulid.test.mjs @@ -0,0 +1,50 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { ulid, ulidTime, isUlid, ULID_RE } from "../dist/index.js"; + +test("ulid shape: 26 chars, Crockford base32, first char 0-7", () => { + for (let i = 0; i < 50; i++) { + const id = ulid(); + assert.equal(id.length, 26); + assert.match(id, ULID_RE); + assert.equal(isUlid(id), true); + } +}); + +test("ulid time round-trip", () => { + const times = [0, 1, 1234567890123, Date.now(), 2 ** 48 - 1]; + for (const t of times) { + assert.equal(ulidTime(ulid(t)), t, `round-trip failed for ${t}`); + } +}); + +test("ulid lexical ordering follows increasing timestamps", () => { + const base = Date.now(); + const ids = []; + for (let i = 0; i < 100; i++) { + ids.push(ulid(base + i * 1000)); + } + const sorted = [...ids].sort(); + assert.deepEqual(sorted, ids); +}); + +test("ulidTime rejects malformed input", () => { + assert.throws(() => ulidTime("not-a-ulid")); + assert.throws(() => ulidTime("")); + // 26 chars but contains excluded letters (I, L, O, U) + assert.throws(() => ulidTime("01ILOU0000000000000000000I")); + // first char out of range (time overflow) + assert.throws(() => ulidTime("81ARZ3NDEKTSV4RRFFQ69G5FAV")); +}); + +test("ulid rejects out-of-range time", () => { + assert.throws(() => ulid(-1)); + assert.throws(() => ulid(2 ** 48)); + assert.throws(() => ulid(1.5)); +}); + +test("isUlid predicate", () => { + assert.equal(isUlid("01ARZ3NDEKTSV4RRFFQ69G5FAV"), true); + assert.equal(isUlid("01arz3ndektsv4rrffq69g5fav"), false); // lowercase not allowed + assert.equal(isUlid("01ARZ3NDEKTSV4RRFFQ69G5FA"), false); // too short +}); diff --git a/intelligence/packages/ontology/test/validate.test.mjs b/intelligence/packages/ontology/test/validate.test.mjs new file mode 100644 index 0000000..6cc7ba7 --- /dev/null +++ b/intelligence/packages/ontology/test/validate.test.mjs @@ -0,0 +1,492 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { createValidators, ulid, METRICS, isKnownMetric } from "../dist/index.js"; + +const v = createValidators(); + +function expectOk(result, label) { + assert.equal(result.ok, true, `${label}: expected valid, got errors: ${result.ok ? "" : result.errors.join("; ")}`); +} + +function expectFail(result, label, needle) { + assert.equal(result.ok, false, `${label}: expected validation failure`); + if (needle !== undefined) { + const joined = result.errors.join("\n"); + assert.match(joined, needle, `${label}: expected error matching ${needle} in:\n${joined}`); + } +} + +// --------------------------------------------------------------------------- +// Schema compilation +// --------------------------------------------------------------------------- + +test("all schemas compile with no unresolved $refs", () => { + // createValidators() eagerly compiles every envelope schema (including the + // cross-file refs into defs/geojson/entity-types); reaching here means all + // $refs resolved. Also sanity-check the registry contents. + const names = [ + "entity", + "relationship", + "observation", + "source", + "gap", + "inference", + "grid", + "feed", + "sduiPanel", + ]; + for (const n of names) { + assert.equal(typeof v[n], "function", `validator ${n} missing`); + } + const base = "https://pact.tailor.au/intelligence/schemas/"; + for (const ref of [ + "defs.schema.json", + "geojson.schema.json", + "entity-types/generator.properties.json", + "entity-types/vessel.properties.json", + ]) { + assert.ok(v.ajv.getSchema(base + ref), `schema not registered: ${ref}`); + } +}); + +// --------------------------------------------------------------------------- +// Entity +// --------------------------------------------------------------------------- + +const validEntity = { + entity_id: "gen:au-nem:bayswater", + entity_type: "Generator", + name: "Bayswater Power Station", + aliases: ["BAYSW"], + data_class: "structural", + observability: "KNOWN_LIVE", + geometry: { type: "Point", coordinates: [150.9484, -32.3946] }, + country: "au", + admin: { region: "NSW", locality: "Muswellbrook" }, + grid_id: "grid:au-nem", + properties: { fuel: "coal", capacity_mw: 2640, unit_count: 4, status: "operating" }, + external_ids: { aemo_duid: ["BW01", "BW02"], aemo_station_id: "BAYSW" }, + sources: ["source:aemo:nem-registration"], + valid_from: "1985-01-01T00:00:00Z", + valid_to: null, + same_as: null, +}; + +test("entity: valid generator passes", () => { + expectOk(v.entity(validEntity), "entity"); +}); + +test("entity: unknown external_ids key fails", () => { + const bad = { ...validEntity, external_ids: { aemo_duid: ["BW01"], nasa_id: "x" } }; + expectFail(v.entity(bad), "entity external_ids", /additional properties.*nasa_id|nasa_id/); +}); + +test("entity: wrong observability enum fails", () => { + const bad = { ...validEntity, observability: "LIVE" }; + expectFail(v.entity(bad), "entity observability", /observability/); +}); + +test("entity: generator sub-schema enforces fuel enum", () => { + const bad = { ...validEntity, properties: { ...validEntity.properties, fuel: "unicorn" } }; + expectFail(v.entity(bad), "entity fuel", /fuel/); +}); + +test("entity: data_class must be structural", () => { + const bad = { ...validEntity, data_class: "live" }; + expectFail(v.entity(bad), "entity data_class"); +}); + +test("entity: unknown top-level key fails (additionalProperties)", () => { + const bad = { ...validEntity, extra_field: true }; + expectFail(v.entity(bad), "entity extra key", /extra_field/); +}); + +test("entity: sub-schemas allow extra property keys (permissive bags)", () => { + const ok = { ...validEntity, properties: { ...validEntity.properties, custom_note: "x" } }; + expectOk(v.entity(ok), "entity extra property-bag key"); +}); + +test("entity: vessel synthetic flag accepted", () => { + const vessel = { + entity_id: "vessel:imo:9700100", + entity_type: "Vessel", + name: "Demo Cape", + data_class: "structural", + observability: "ESTIMATED", + properties: { vessel_type: "bulk-carrier", dwt: 180000, synthetic: true }, + sources: ["source:demo:vessels"], + }; + expectOk(v.entity(vessel), "vessel entity"); + const bad = { ...vessel, properties: { ...vessel.properties, synthetic: "yes" } }; + expectFail(v.entity(bad), "vessel synthetic type", /synthetic/); +}); + +// --------------------------------------------------------------------------- +// Relationship +// --------------------------------------------------------------------------- + +const validRel = { + rel_id: "rel:bayswater-owned-by-agl", + rel_type: "OWNED_BY", + from_id: "gen:au-nem:bayswater", + to_id: "company:au:agl", + confidence: 0.95, + source: "source:aemo:nem-registration", + method: "declared", +}; + +test("relationship: valid declared edge passes", () => { + expectOk(v.relationship(validRel), "relationship"); +}); + +test("relationship: inferred edge with evidence passes", () => { + const ok = { + ...validRel, + rel_id: "rel:tomago-may-supply", + rel_type: "MAY_SUPPLY", + method: "inferred", + evidence: [{ kind: "observation", ref: "01ARZ3NDEKTSV4RRFFQ69G5FAV" }], + }; + expectOk(v.relationship(ok), "relationship inferred"); +}); + +test("relationship: method=inferred without evidence fails", () => { + const bad = { ...validRel, method: "inferred" }; + expectFail(v.relationship(bad), "relationship missing evidence", /evidence/); +}); + +test("relationship: method=inferred with empty evidence fails", () => { + const bad = { ...validRel, method: "inferred", evidence: [] }; + expectFail(v.relationship(bad), "relationship empty evidence", /fewer than 1 items|minItems|must NOT have fewer/); +}); + +test("relationship: unknown rel_type fails", () => { + const bad = { ...validRel, rel_type: "FRIENDS_WITH" }; + expectFail(v.relationship(bad), "relationship rel_type", /rel_type/); +}); + +test("relationship: confidence out of range fails", () => { + const bad = { ...validRel, confidence: 1.5 }; + expectFail(v.relationship(bad), "relationship confidence", /confidence/); +}); + +// --------------------------------------------------------------------------- +// Observation +// --------------------------------------------------------------------------- + +const validObs = { + observation_id: ulid(), + entity_id: "gen:au-nem:bayswater", + metric: "power.output.mw", + value: 1980.5, + event_time: "2026-08-12T00:05:00Z", + ingest_time: "2026-08-12T00:05:04Z", + source_id: "source:aemo:nem-dispatch", + feed_id: "feed:aemo:scada", + quality: "good", + unit: "MW", +}; + +test("observation: valid scalar observation passes", () => { + expectOk(v.observation(validObs), "observation"); +}); + +test("observation: object value (vessel.position) passes", () => { + const ok = { + ...validObs, + observation_id: ulid(), + entity_id: "vessel:imo:9700100", + metric: "vessel.position", + value: { lat: -32.9, lon: 151.78, speed_kn: 11.2, heading_deg: 44 }, + unit: undefined, + }; + delete ok.unit; + expectOk(v.observation(ok), "observation object value"); +}); + +test("observation: bad ULID fails", () => { + const bad = { ...validObs, observation_id: "not-a-ulid" }; + expectFail(v.observation(bad), "observation ulid", /observation_id/); +}); + +test("observation: wrong quality enum fails", () => { + const bad = { ...validObs, quality: "excellent" }; + expectFail(v.observation(bad), "observation quality", /quality/); +}); + +test("observation: boolean value fails (must be number|string|object)", () => { + const bad = { ...validObs, value: true }; + expectFail(v.observation(bad), "observation value type", /value/); +}); + +test("observation: bad metric grammar fails", () => { + const bad = { ...validObs, metric: "PowerOutput" }; + expectFail(v.observation(bad), "observation metric", /metric/); +}); + +// --------------------------------------------------------------------------- +// Source +// --------------------------------------------------------------------------- + +const validSource = { + source_id: "source:aemo:nem-dispatch", + publisher: "AEMO", + dataset: "NEM dispatch SCADA", + api_type: "rest", + url: "https://visualisations.aemo.com.au/aemo/apps/api/report/5MIN", + coverage: { grids: ["grid:au-nem"], countries: ["au"] }, + data_class: "live", + licensing: { class: "open-attribution", notes: "AEMO data licence" }, + verification_status: "verified_live", + last_checked: "2026-08-12", + reliability: 5, +}; + +test("source: valid source passes", () => { + expectOk(v.source(validSource), "source"); +}); + +test("source: unknown api_type fails", () => { + const bad = { ...validSource, api_type: "grpc" }; + expectFail(v.source(bad), "source api_type", /api_type/); +}); + +test("source: bad last_checked date fails", () => { + const bad = { ...validSource, last_checked: "12/08/2026" }; + expectFail(v.source(bad), "source date", /last_checked/); +}); + +test("source: reliability out of 1-5 fails", () => { + const bad = { ...validSource, reliability: 7 }; + expectFail(v.source(bad), "source reliability", /reliability/); +}); + +// --------------------------------------------------------------------------- +// Observability gap +// --------------------------------------------------------------------------- + +const validGap = { + gap_id: "gap:au-nem:tomago-load", + entity_id: "smelter:au:tomago", + desired_metric: "power.load.mw", + commercial_value: "high", + strategic_value: "critical", + priority: "P1", + status: "open", + current_source: null, + best_available_proxy: { + source_id: "source:aemo:nem-dispatch", + metric: "grid.demand.mw", + derivation: "regional demand residual after known generation", + }, + instrumentation_options: [ + { kind: "ct_sensor", notes: "clamp at 330kV feeder", indicative_cost_band: "medium" }, + { kind: "partner_feed" }, + ], + estimated_complexity: "medium", +}; + +test("gap: valid gap passes", () => { + expectOk(v.gap(validGap), "gap"); +}); + +test("gap: bad priority fails", () => { + const bad = { ...validGap, priority: "P5" }; + expectFail(v.gap(bad), "gap priority", /priority/); +}); + +test("gap: unknown instrumentation kind fails", () => { + const bad = { + ...validGap, + instrumentation_options: [{ kind: "drone" }], + }; + expectFail(v.gap(bad), "gap instrumentation kind", /kind/); +}); + +test("gap: bad desired_metric grammar fails", () => { + const bad = { ...validGap, desired_metric: "load" }; + expectFail(v.gap(bad), "gap metric", /desired_metric/); +}); + +// --------------------------------------------------------------------------- +// Inference +// --------------------------------------------------------------------------- + +const validInference = { + inference_id: "infer:2026-08-12:tomago-potline-restart", + claim: "Tomago potline 2 restarted around 2026-08-11T22:00Z", + claim_structured: { + subject: "smelter:au:tomago", + predicate: "load_step_up", + object: "potline-2", + qualifiers: { magnitude_mw: 280 }, + }, + tier: "B", + confidence: 0.72, + status: "INFERRED", + method: "rule:load-step@1.0.0", + evidence: [ + { kind: "observation", ref: "01ARZ3NDEKTSV4RRFFQ69G5FAV", role: "supports" }, + { kind: "structural", ref: "gen:au-nem:bayswater", role: "supports" }, + ], + contrary_evidence: [ + { kind: "observation", ref: "01ARZ3NDEKTSV4RRFFQ69G5FAW", role: "contradicts" }, + ], + sources: ["source:aemo:nem-dispatch"], + produced_at: "2026-08-12T01:00:00Z", + event_time_range: { from: "2026-08-11T21:00:00Z", to: "2026-08-11T23:00:00Z" }, + lineage: { correlation_id: "corr-1", in_response_to: null, prev_hash: null }, + pact: null, +}; + +test("inference: valid inference passes", () => { + expectOk(v.inference(validInference), "inference"); +}); + +test("inference: empty evidence array fails", () => { + const bad = { ...validInference, evidence: [] }; + expectFail(v.inference(bad), "inference empty evidence", /evidence/); +}); + +test("inference: bad method pattern fails", () => { + const bad = { ...validInference, method: "load-step" }; + expectFail(v.inference(bad), "inference method", /method/); +}); + +test("inference: evidence role must be supports", () => { + const bad = { + ...validInference, + evidence: [{ kind: "observation", ref: "01ARZ3NDEKTSV4RRFFQ69G5FAV", role: "contradicts" }], + }; + expectFail(v.inference(bad), "inference role", /role/); +}); + +test("inference: bad inference_id fails", () => { + const bad = { ...validInference, inference_id: "infer:tomago" }; + expectFail(v.inference(bad), "inference id", /inference_id/); +}); + +// --------------------------------------------------------------------------- +// Grid registry +// --------------------------------------------------------------------------- + +const validGrid = { + grid_id: "grid:au-nem", + name: "National Electricity Market", + kind: "market", + countries: ["au"], + frequency_hz: 50, + live_data: { + availability: "rich", + notes: "5-minute dispatch, SCADA per DUID", + source_ids: ["source:aemo:nem-dispatch"], + }, + sources: ["source:aemo:nem-registration"], + parent_grid: null, + operators: [ + { name: "AEMO", role: "market_operator" }, + { name: "Transgrid", role: "tso" }, + ], + market: { structure: "gross pool", price_mechanism: "regional marginal", dispatch_interval: "5m" }, + timezone: "Australia/Brisbane", + interconnections: [{ to_grid: "grid:au-swis", name: "none", capacity_mw: null }], + demand_range_mw: { min: 14000, max: 35000 }, +}; + +test("grid: valid registry record passes", () => { + expectOk(v.grid(validGrid), "grid"); +}); + +test("grid: frequency must be 50 or 60", () => { + const bad = { ...validGrid, frequency_hz: 55 }; + expectFail(v.grid(bad), "grid frequency", /frequency_hz/); +}); + +test("grid: unknown kind fails", () => { + const bad = { ...validGrid, kind: "mega_grid" }; + expectFail(v.grid(bad), "grid kind", /kind/); +}); + +test("grid: empty countries fails", () => { + const bad = { ...validGrid, countries: [] }; + expectFail(v.grid(bad), "grid countries", /countries/); +}); + +// --------------------------------------------------------------------------- +// Telemetry feed +// --------------------------------------------------------------------------- + +const validFeed = { + feed_id: "feed:aemo:scada", + source_id: "source:aemo:nem-dispatch", + connector_id: "aemo-nemweb", + metrics: ["power.output.mw", "grid.demand.mw"], + cadence: "5m", + description: "NEM dispatch SCADA per DUID", +}; + +test("feed: valid feed passes", () => { + expectOk(v.feed(validFeed), "feed"); +}); + +test("feed: empty metrics fails", () => { + const bad = { ...validFeed, metrics: [] }; + expectFail(v.feed(bad), "feed metrics", /metrics/); +}); + +test("feed: malformed feed_id fails", () => { + const bad = { ...validFeed, feed_id: "aemo:scada" }; + expectFail(v.feed(bad), "feed id", /feed_id/); +}); + +// --------------------------------------------------------------------------- +// SDUI panel +// --------------------------------------------------------------------------- + +const validSdui = { + entity_id: "gen:au-nem:bayswater", + entity_type: "Generator", + situation: ["normal"], + layout: [ + { panel: "headline-state" }, + { + panel: "timeseries", + props: { metric: "power.output.mw" }, + data: { endpoint: "/api/intel/entities/gen:au-nem:bayswater/timeseries" }, + }, + ], +}; + +test("sduiPanel: valid doc passes", () => { + expectOk(v.sduiPanel(validSdui), "sdui"); +}); + +test("sduiPanel: endpoint not under /api/intel/ fails", () => { + const bad = { + ...validSdui, + layout: [{ panel: "timeseries", data: { endpoint: "/api/other/x" } }], + }; + expectFail(v.sduiPanel(bad), "sdui endpoint", /endpoint/); +}); + +test("sduiPanel: unknown panel kind fails", () => { + const bad = { ...validSdui, layout: [{ panel: "pie-chart" }] }; + expectFail(v.sduiPanel(bad), "sdui panel kind", /panel/); +}); + +test("sduiPanel: empty layout fails", () => { + const bad = { ...validSdui, layout: [] }; + expectFail(v.sduiPanel(bad), "sdui layout", /layout/); +}); + +// --------------------------------------------------------------------------- +// Metrics vocabulary +// --------------------------------------------------------------------------- + +test("metrics: canonical vocabulary present and helper works", () => { + assert.equal(Object.keys(METRICS).length, 20); + assert.equal(isKnownMetric("power.output.mw"), true); + assert.equal(isKnownMetric("vessel.position"), true); + assert.equal(isKnownMetric("made.up.metric"), false); + assert.equal(METRICS["market.price.energy"].unit, "AUD/MWh"); +}); diff --git a/intelligence/schemas/defs.schema.json b/intelligence/schemas/defs.schema.json new file mode 100644 index 0000000..cd52d1a --- /dev/null +++ b/intelligence/schemas/defs.schema.json @@ -0,0 +1,119 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/defs.schema.json", + "title": "Shared definitions", + "description": "Common $defs shared by all PACT.TAILOR intelligence schemas.", + "$defs": { + "timestamp": { + "type": "string", + "format": "date-time", + "description": "ISO 8601 timestamp" + }, + "nullableTimestamp": { + "anyOf": [ + { "$ref": "#/$defs/timestamp" }, + { "type": "null" } + ] + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1, + "description": "Confidence in [0, 1]" + }, + "entityId": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*:[a-z0-9][a-z0-9-]*(:[a-z0-9][a-z0-9-]*)?$", + "description": "typeCode:namespace[:slug]" + }, + "relId": { + "type": "string", + "pattern": "^rel:[a-z0-9][a-z0-9-]*$" + }, + "sourceId": { + "type": "string", + "pattern": "^source:[a-z0-9-]+:[a-z0-9-]+$" + }, + "gapId": { + "type": "string", + "pattern": "^gap:[a-z0-9-]+:[a-z0-9-]+$" + }, + "feedId": { + "type": "string", + "pattern": "^feed:[a-z0-9-]+:[a-z0-9-]+(:[a-z0-9-]+)?$" + }, + "inferenceId": { + "type": "string", + "pattern": "^infer:[0-9]{4}-[0-9]{2}-[0-9]{2}:[a-z0-9][a-z0-9-]*$" + }, + "ulid": { + "type": "string", + "pattern": "^[0-7][0-9A-HJKMNP-TV-Z]{25}$", + "description": "ULID (Crockford base32, 26 chars)" + }, + "metric": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]*(\\.[a-z0-9_]+)+$", + "description": "Dotted lowercase metric name, e.g. power.output.mw" + }, + "countryCode": { + "type": "string", + "pattern": "^[a-z]{2}$", + "description": "ISO 3166-1 alpha-2, lowercase" + }, + "entityType": { + "enum": [ + "Grid", + "GridRegion", + "Operator", + "Generator", + "StorageAsset", + "TransmissionAsset", + "Interconnector", + "Substation", + "CoalBasin", + "CoalMine", + "GasBasin", + "GasField", + "GasProcessingPlant", + "Pipeline", + "HydroReservoir", + "IndustrialFacility", + "Smelter", + "SteelMill", + "Mine", + "Refinery", + "LNGPlant", + "ChemicalPlant", + "DataCentre", + "Port", + "Terminal", + "Berth", + "Railway", + "Train", + "Vessel", + "Company", + "Commodity", + "Product", + "Market", + "Destination", + "Sensor", + "TelemetryFeed", + "Observation", + "Source", + "Evidence", + "Inference", + "ObservabilityGap" + ] + }, + "observability": { + "enum": [ + "KNOWN_LIVE", + "ESTIMATED", + "NOT_OBSERVABLE", + "INSTRUMENTATION_OPPORTUNITY", + "UNCLASSIFIED" + ] + } + } +} diff --git a/intelligence/schemas/entity-types/generator.properties.json b/intelligence/schemas/entity-types/generator.properties.json new file mode 100644 index 0000000..cf99269 --- /dev/null +++ b/intelligence/schemas/entity-types/generator.properties.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/entity-types/generator.properties.json", + "title": "Generator properties", + "type": "object", + "properties": { + "fuel": { + "enum": ["coal", "gas", "hydro", "wind", "solar", "battery", "pumped-hydro", "diesel", "biomass", "nuclear", "other"] + }, + "capacity_mw": { "type": "number" }, + "technology": { "type": "string" }, + "commissioned": { "type": "integer", "minimum": 1800, "maximum": 2200 }, + "unit_count": { "type": "integer" }, + "status": { + "enum": ["operating", "mothballed", "retired", "announced", "construction"] + } + }, + "additionalProperties": true +} diff --git a/intelligence/schemas/entity-types/grid.properties.json b/intelligence/schemas/entity-types/grid.properties.json new file mode 100644 index 0000000..1350c61 --- /dev/null +++ b/intelligence/schemas/entity-types/grid.properties.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/entity-types/grid.properties.json", + "title": "Grid properties", + "type": "object", + "properties": { + "frequency_hz": { "enum": [50, 60] }, + "kind": { "type": "string" } + }, + "additionalProperties": true +} diff --git a/intelligence/schemas/entity-types/interconnector.properties.json b/intelligence/schemas/entity-types/interconnector.properties.json new file mode 100644 index 0000000..2495ba2 --- /dev/null +++ b/intelligence/schemas/entity-types/interconnector.properties.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/entity-types/interconnector.properties.json", + "title": "Interconnector properties", + "type": "object", + "properties": { + "capacity_mw_forward": { "type": "number" }, + "capacity_mw_reverse": { "type": "number" }, + "from_region": { "type": "string" }, + "to_region": { "type": "string" }, + "hvdc": { "type": "boolean" } + }, + "additionalProperties": true +} diff --git a/intelligence/schemas/entity-types/mine.properties.json b/intelligence/schemas/entity-types/mine.properties.json new file mode 100644 index 0000000..33ed5bb --- /dev/null +++ b/intelligence/schemas/entity-types/mine.properties.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/entity-types/mine.properties.json", + "title": "Mine properties (Mine and CoalMine)", + "type": "object", + "properties": { + "commodities": { + "type": "array", + "items": { "type": "string" } + }, + "mine_type": { + "enum": ["open-cut", "underground", "mixed"] + }, + "capacity_mtpa": { "type": "number" }, + "status": { "type": "string" } + }, + "additionalProperties": true +} diff --git a/intelligence/schemas/entity-types/pipeline.properties.json b/intelligence/schemas/entity-types/pipeline.properties.json new file mode 100644 index 0000000..938b784 --- /dev/null +++ b/intelligence/schemas/entity-types/pipeline.properties.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/entity-types/pipeline.properties.json", + "title": "Pipeline properties", + "type": "object", + "properties": { + "commodity": { + "enum": ["gas", "oil", "water", "slurry"] + }, + "length_km": { "type": "number" }, + "capacity_tj_day": { "type": "number" } + }, + "additionalProperties": true +} diff --git a/intelligence/schemas/entity-types/port.properties.json b/intelligence/schemas/entity-types/port.properties.json new file mode 100644 index 0000000..70d9290 --- /dev/null +++ b/intelligence/schemas/entity-types/port.properties.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/entity-types/port.properties.json", + "title": "Port properties", + "type": "object", + "properties": { + "harbour_type": { "type": "string" }, + "max_draught_m": { "type": "number" } + }, + "additionalProperties": true +} diff --git a/intelligence/schemas/entity-types/smelter.properties.json b/intelligence/schemas/entity-types/smelter.properties.json new file mode 100644 index 0000000..e62a2d2 --- /dev/null +++ b/intelligence/schemas/entity-types/smelter.properties.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/entity-types/smelter.properties.json", + "title": "Smelter properties", + "type": "object", + "properties": { + "product": { "type": "string" }, + "capacity_tpa": { "type": "number" }, + "potlines": { "type": "integer" }, + "technology": { "type": "string" } + }, + "additionalProperties": true +} diff --git a/intelligence/schemas/entity-types/terminal.properties.json b/intelligence/schemas/entity-types/terminal.properties.json new file mode 100644 index 0000000..29e7e55 --- /dev/null +++ b/intelligence/schemas/entity-types/terminal.properties.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/entity-types/terminal.properties.json", + "title": "Terminal properties", + "type": "object", + "properties": { + "commodities": { + "type": "array", + "items": { "type": "string" } + }, + "nameplate_mtpa": { "type": "number" } + }, + "additionalProperties": true +} diff --git a/intelligence/schemas/entity-types/vessel.properties.json b/intelligence/schemas/entity-types/vessel.properties.json new file mode 100644 index 0000000..edb3d40 --- /dev/null +++ b/intelligence/schemas/entity-types/vessel.properties.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/entity-types/vessel.properties.json", + "title": "Vessel properties", + "type": "object", + "properties": { + "vessel_type": { "type": "string" }, + "dwt": { "type": "number" }, + "loa_m": { "type": "number" }, + "beam_m": { "type": "number" }, + "flag": { "type": "string" }, + "synthetic": { + "type": "boolean", + "description": "true marks exemplar/demo vessels that are NOT real ships" + } + }, + "additionalProperties": true +} diff --git a/intelligence/schemas/entity.schema.json b/intelligence/schemas/entity.schema.json new file mode 100644 index 0000000..120768a --- /dev/null +++ b/intelligence/schemas/entity.schema.json @@ -0,0 +1,129 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/entity.schema.json", + "title": "Entity (class A structural)", + "description": "Structural entity envelope for the PACT.TAILOR intelligence graph.", + "type": "object", + "required": ["entity_id", "entity_type", "name", "data_class", "observability", "sources"], + "properties": { + "entity_id": { "$ref": "defs.schema.json#/$defs/entityId" }, + "entity_type": { "$ref": "defs.schema.json#/$defs/entityType" }, + "name": { "type": "string" }, + "aliases": { + "type": "array", + "items": { "type": "string" } + }, + "data_class": { "const": "structural" }, + "observability": { "$ref": "defs.schema.json#/$defs/observability" }, + "geometry": { + "anyOf": [ + { "$ref": "geojson.schema.json#/$defs/geometry" }, + { "type": "null" } + ] + }, + "country": { + "anyOf": [ + { "$ref": "defs.schema.json#/$defs/countryCode" }, + { "type": "null" } + ] + }, + "admin": { + "type": "object", + "properties": { + "region": { "type": "string" }, + "locality": { "type": "string" } + }, + "additionalProperties": false + }, + "grid_id": { + "anyOf": [ + { "$ref": "defs.schema.json#/$defs/entityId" }, + { "type": "null" } + ] + }, + "properties": { + "type": "object", + "description": "Per-entity-type properties; typed sub-schemas apply via the allOf conditionals below." + }, + "external_ids": { + "type": "object", + "properties": { + "aemo_duid": { + "type": "array", + "items": { "type": "string" } + }, + "aemo_station_id": { "type": "string" }, + "aemo_region_id": { "type": "string" }, + "entsoe_eic": { "type": "string" }, + "eia_plant_code": { "type": "string" }, + "gem_id": { "type": "string" }, + "imo": { "type": "string" }, + "mmsi": { "type": "string" }, + "callsign": { "type": "string" }, + "unlocode": { "type": "string" }, + "abn": { "type": "string" }, + "lei": { "type": "string" }, + "wikidata_qid": { "type": "string" }, + "osm_id": { "type": "string" } + }, + "additionalProperties": false + }, + "sources": { + "type": "array", + "items": { "$ref": "defs.schema.json#/$defs/sourceId" }, + "minItems": 1 + }, + "valid_from": { "$ref": "defs.schema.json#/$defs/nullableTimestamp" }, + "valid_to": { "$ref": "defs.schema.json#/$defs/nullableTimestamp" }, + "notes": { "type": "string" }, + "same_as": { + "anyOf": [ + { "$ref": "defs.schema.json#/$defs/entityId" }, + { "type": "null" } + ] + } + }, + "allOf": [ + { + "if": { "properties": { "entity_type": { "const": "Generator" } }, "required": ["entity_type"] }, + "then": { "properties": { "properties": { "$ref": "entity-types/generator.properties.json" } } } + }, + { + "if": { "properties": { "entity_type": { "const": "Interconnector" } }, "required": ["entity_type"] }, + "then": { "properties": { "properties": { "$ref": "entity-types/interconnector.properties.json" } } } + }, + { + "if": { "properties": { "entity_type": { "const": "Vessel" } }, "required": ["entity_type"] }, + "then": { "properties": { "properties": { "$ref": "entity-types/vessel.properties.json" } } } + }, + { + "if": { "properties": { "entity_type": { "const": "Port" } }, "required": ["entity_type"] }, + "then": { "properties": { "properties": { "$ref": "entity-types/port.properties.json" } } } + }, + { + "if": { "properties": { "entity_type": { "const": "Terminal" } }, "required": ["entity_type"] }, + "then": { "properties": { "properties": { "$ref": "entity-types/terminal.properties.json" } } } + }, + { + "if": { "properties": { "entity_type": { "const": "Smelter" } }, "required": ["entity_type"] }, + "then": { "properties": { "properties": { "$ref": "entity-types/smelter.properties.json" } } } + }, + { + "if": { "properties": { "entity_type": { "const": "Mine" } }, "required": ["entity_type"] }, + "then": { "properties": { "properties": { "$ref": "entity-types/mine.properties.json" } } } + }, + { + "if": { "properties": { "entity_type": { "const": "CoalMine" } }, "required": ["entity_type"] }, + "then": { "properties": { "properties": { "$ref": "entity-types/mine.properties.json" } } } + }, + { + "if": { "properties": { "entity_type": { "const": "Pipeline" } }, "required": ["entity_type"] }, + "then": { "properties": { "properties": { "$ref": "entity-types/pipeline.properties.json" } } } + }, + { + "if": { "properties": { "entity_type": { "const": "Grid" } }, "required": ["entity_type"] }, + "then": { "properties": { "properties": { "$ref": "entity-types/grid.properties.json" } } } + } + ], + "additionalProperties": false +} diff --git a/intelligence/schemas/geojson.schema.json b/intelligence/schemas/geojson.schema.json new file mode 100644 index 0000000..266c245 --- /dev/null +++ b/intelligence/schemas/geojson.schema.json @@ -0,0 +1,79 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/geojson.schema.json", + "title": "GeoJSON (minimal WGS84 subset)", + "description": "Minimal GeoJSON geometry definitions. Positions are [lon, lat] (optionally + elevation) number arrays in WGS84.", + "$defs": { + "position": { + "type": "array", + "items": { "type": "number" }, + "minItems": 2, + "maxItems": 3, + "description": "[lon, lat] or [lon, lat, elevation]" + }, + "linearRing": { + "type": "array", + "items": { "$ref": "#/$defs/position" }, + "minItems": 4 + }, + "point": { + "type": "object", + "required": ["type", "coordinates"], + "properties": { + "type": { "const": "Point" }, + "coordinates": { "$ref": "#/$defs/position" } + }, + "additionalProperties": false + }, + "lineString": { + "type": "object", + "required": ["type", "coordinates"], + "properties": { + "type": { "const": "LineString" }, + "coordinates": { + "type": "array", + "items": { "$ref": "#/$defs/position" }, + "minItems": 2 + } + }, + "additionalProperties": false + }, + "polygon": { + "type": "object", + "required": ["type", "coordinates"], + "properties": { + "type": { "const": "Polygon" }, + "coordinates": { + "type": "array", + "items": { "$ref": "#/$defs/linearRing" }, + "minItems": 1 + } + }, + "additionalProperties": false + }, + "multiPolygon": { + "type": "object", + "required": ["type", "coordinates"], + "properties": { + "type": { "const": "MultiPolygon" }, + "coordinates": { + "type": "array", + "items": { + "type": "array", + "items": { "$ref": "#/$defs/linearRing" }, + "minItems": 1 + } + } + }, + "additionalProperties": false + }, + "geometry": { + "oneOf": [ + { "$ref": "#/$defs/point" }, + { "$ref": "#/$defs/lineString" }, + { "$ref": "#/$defs/polygon" }, + { "$ref": "#/$defs/multiPolygon" } + ] + } + } +} diff --git a/intelligence/schemas/grid.schema.json b/intelligence/schemas/grid.schema.json new file mode 100644 index 0000000..0c41a05 --- /dev/null +++ b/intelligence/schemas/grid.schema.json @@ -0,0 +1,120 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/grid.schema.json", + "title": "Grid registry record", + "description": "Registry record for a power grid / synchronous area / market (richer than a Grid entity).", + "type": "object", + "required": ["grid_id", "name", "kind", "countries", "frequency_hz", "live_data", "sources"], + "properties": { + "grid_id": { + "type": "string", + "pattern": "^grid:[a-z0-9-]+$" + }, + "name": { "type": "string" }, + "kind": { + "enum": [ + "synchronous_area", + "interconnection", + "iso_rto", + "tso_area", + "market", + "balancing_authority", + "national_grid", + "regional_grid", + "isolated_grid", + "captive" + ] + }, + "countries": { + "type": "array", + "items": { "$ref": "defs.schema.json#/$defs/countryCode" }, + "minItems": 1 + }, + "frequency_hz": { "enum": [50, 60] }, + "live_data": { + "type": "object", + "required": ["availability"], + "properties": { + "availability": { + "enum": ["rich", "partial", "minimal", "none"] + }, + "notes": { "type": "string" }, + "source_ids": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": false + }, + "sources": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "parent_grid": { + "anyOf": [ + { "type": "string", "pattern": "^grid:[a-z0-9-]+$" }, + { "type": "null" } + ] + }, + "operators": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "role"], + "properties": { + "name": { "type": "string" }, + "role": { + "enum": ["system_operator", "market_operator", "tso", "transmission_owner", "regulator"] + } + }, + "additionalProperties": false + } + }, + "market": { + "anyOf": [ + { + "type": "object", + "properties": { + "structure": { "type": "string" }, + "price_mechanism": { "type": "string" }, + "dispatch_interval": { "type": "string" } + }, + "additionalProperties": false + }, + { "type": "null" } + ] + }, + "timezone": { "type": "string" }, + "interconnections": { + "type": "array", + "items": { + "type": "object", + "required": ["to_grid"], + "properties": { + "to_grid": { "type": "string" }, + "name": { "type": "string" }, + "capacity_mw": { + "type": ["number", "null"] + } + }, + "additionalProperties": false + } + }, + "demand_range_mw": { + "anyOf": [ + { + "type": "object", + "properties": { + "min": { "type": "number" }, + "max": { "type": "number" } + }, + "additionalProperties": false + }, + { "type": "null" } + ] + }, + "notes": { "type": "string" } + }, + "additionalProperties": false +} diff --git a/intelligence/schemas/inference.schema.json b/intelligence/schemas/inference.schema.json new file mode 100644 index 0000000..80097ab --- /dev/null +++ b/intelligence/schemas/inference.schema.json @@ -0,0 +1,103 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/inference.schema.json", + "title": "Inference (class C)", + "description": "A derived claim with evidence, lineage, and confidence.", + "type": "object", + "required": ["inference_id", "claim", "claim_structured", "tier", "confidence", "status", "method", "evidence", "sources", "produced_at"], + "properties": { + "inference_id": { "$ref": "defs.schema.json#/$defs/inferenceId" }, + "claim": { "type": "string" }, + "claim_structured": { + "type": "object", + "required": ["subject", "predicate", "object"], + "properties": { + "subject": { "$ref": "defs.schema.json#/$defs/entityId" }, + "predicate": { "type": "string" }, + "object": { "type": "string" }, + "qualifiers": { "type": "object" } + }, + "additionalProperties": false + }, + "tier": { + "enum": ["A", "B", "C"] + }, + "confidence": { "$ref": "defs.schema.json#/$defs/confidence" }, + "status": { + "enum": ["INFERRED", "CORROBORATED", "CONTESTED", "VERIFIED", "RETRACTED"] + }, + "method": { + "type": "string", + "pattern": "^rule:[a-z0-9-]+@[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "evidence": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["kind", "ref", "role"], + "properties": { + "kind": { "enum": ["observation", "structural", "inference"] }, + "ref": { "type": "string" }, + "role": { "const": "supports" } + }, + "additionalProperties": false + } + }, + "contrary_evidence": { + "type": "array", + "items": { + "type": "object", + "required": ["kind", "ref", "role"], + "properties": { + "kind": { "enum": ["observation", "structural", "inference"] }, + "ref": { "type": "string" }, + "role": { "const": "contradicts" } + }, + "additionalProperties": false + } + }, + "sources": { + "type": "array", + "items": { "$ref": "defs.schema.json#/$defs/sourceId" }, + "minItems": 1 + }, + "produced_at": { "$ref": "defs.schema.json#/$defs/timestamp" }, + "event_time_range": { + "type": "object", + "properties": { + "from": { "$ref": "defs.schema.json#/$defs/timestamp" }, + "to": { "$ref": "defs.schema.json#/$defs/timestamp" } + }, + "additionalProperties": false + }, + "lineage": { + "type": "object", + "properties": { + "correlation_id": { "type": "string" }, + "in_response_to": { + "type": ["string", "null"] + }, + "prev_hash": { + "type": ["string", "null"] + } + }, + "additionalProperties": false + }, + "pact": { + "anyOf": [ + { + "type": "object", + "properties": { + "fabric_id": { "type": "string" }, + "proposal_id": { "type": "string" } + }, + "additionalProperties": false + }, + { "type": "null" } + ] + }, + "notes": { "type": "string" } + }, + "additionalProperties": false +} diff --git a/intelligence/schemas/observability-gap.schema.json b/intelligence/schemas/observability-gap.schema.json new file mode 100644 index 0000000..aaebab0 --- /dev/null +++ b/intelligence/schemas/observability-gap.schema.json @@ -0,0 +1,91 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/observability-gap.schema.json", + "title": "Observability gap", + "description": "A desired-but-missing metric for an entity, with instrumentation options.", + "type": "object", + "required": ["gap_id", "entity_id", "desired_metric", "commercial_value", "strategic_value", "priority", "status"], + "properties": { + "gap_id": { "$ref": "defs.schema.json#/$defs/gapId" }, + "entity_id": { "$ref": "defs.schema.json#/$defs/entityId" }, + "desired_metric": { "$ref": "defs.schema.json#/$defs/metric" }, + "commercial_value": { + "enum": ["low", "medium", "high", "critical"] + }, + "strategic_value": { + "enum": ["low", "medium", "high", "critical"] + }, + "priority": { + "enum": ["P1", "P2", "P3", "P4"] + }, + "status": { + "enum": ["open", "proxied", "instrumented", "wont_do"] + }, + "current_source": { + "anyOf": [ + { "$ref": "defs.schema.json#/$defs/sourceId" }, + { "type": "null" } + ] + }, + "best_available_proxy": { + "anyOf": [ + { + "type": "object", + "required": ["source_id", "metric"], + "properties": { + "source_id": { "$ref": "defs.schema.json#/$defs/sourceId" }, + "metric": { "$ref": "defs.schema.json#/$defs/metric" }, + "derivation": { "type": "string" } + }, + "additionalProperties": false + }, + { "type": "null" } + ] + }, + "required_resolution": { "type": "string" }, + "required_accuracy": { "type": "string" }, + "instrumentation_options": { + "type": "array", + "items": { + "type": "object", + "required": ["kind"], + "properties": { + "kind": { + "enum": [ + "ct_sensor", + "revenue_meter", + "power_quality_meter", + "substation_gateway", + "iot_gateway", + "weather_station", + "camera_ocr", + "ais_receiver", + "gnss", + "scada_integration", + "plc_integration", + "opc_ua", + "modbus", + "mqtt_gateway", + "environmental_sensor", + "condition_sensor", + "satellite_thermal", + "satellite_optical", + "partner_feed", + "customer_api" + ] + }, + "notes": { "type": "string" }, + "indicative_cost_band": { + "enum": ["low", "medium", "high"] + } + }, + "additionalProperties": false + } + }, + "estimated_complexity": { + "enum": ["low", "medium", "high"] + }, + "notes": { "type": "string" } + }, + "additionalProperties": false +} diff --git a/intelligence/schemas/observation.schema.json b/intelligence/schemas/observation.schema.json new file mode 100644 index 0000000..fef3ade --- /dev/null +++ b/intelligence/schemas/observation.schema.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/observation.schema.json", + "title": "Observation (class B)", + "description": "A single time-stamped metric observation about an entity.", + "type": "object", + "required": ["observation_id", "entity_id", "metric", "value", "event_time", "ingest_time", "source_id", "feed_id", "quality"], + "properties": { + "observation_id": { "$ref": "defs.schema.json#/$defs/ulid" }, + "entity_id": { "$ref": "defs.schema.json#/$defs/entityId" }, + "metric": { "$ref": "defs.schema.json#/$defs/metric" }, + "value": { + "type": ["number", "string", "object"], + "description": "Scalar value, or an object for structured metrics (e.g. vessel.position {lat, lon, speed_kn, heading_deg, draught_m?})" + }, + "event_time": { "$ref": "defs.schema.json#/$defs/timestamp" }, + "ingest_time": { "$ref": "defs.schema.json#/$defs/timestamp" }, + "source_id": { "$ref": "defs.schema.json#/$defs/sourceId" }, + "feed_id": { "$ref": "defs.schema.json#/$defs/feedId" }, + "quality": { + "enum": ["good", "suspect", "estimated", "interpolated", "missing"] + }, + "unit": { "type": "string" }, + "is_correction": { "type": "boolean", "default": false }, + "corrects": { + "anyOf": [ + { "$ref": "defs.schema.json#/$defs/ulid" }, + { "type": "null" } + ] + }, + "source_sequence": { "type": "string" }, + "meta": { "type": "object" } + }, + "additionalProperties": false +} diff --git a/intelligence/schemas/relationship.schema.json b/intelligence/schemas/relationship.schema.json new file mode 100644 index 0000000..2560f1b --- /dev/null +++ b/intelligence/schemas/relationship.schema.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/relationship.schema.json", + "title": "Relationship", + "description": "Directed, sourced, confidence-weighted edge between two entities.", + "type": "object", + "required": ["rel_id", "rel_type", "from_id", "to_id", "confidence", "source", "method"], + "properties": { + "rel_id": { "$ref": "defs.schema.json#/$defs/relId" }, + "rel_type": { + "enum": [ + "PART_OF", + "CONNECTED_TO", + "INTERCONNECTED_WITH", + "OWNED_BY", + "OPERATED_BY", + "EXTRACTS", + "SUPPLIES", + "MAY_SUPPLY", + "TRANSPORTS", + "GENERATES", + "TRANSMITS", + "IMPORTS_FROM", + "EXPORTS_TO", + "CONSUMES", + "PRODUCES", + "LOADS_AT", + "DEPARTS_FROM", + "ARRIVES_AT", + "SHIPS_TO", + "OBSERVED_BY", + "REPORTED_BY", + "INFERRED_FROM", + "SUPPORTED_BY", + "CONTRADICTED_BY" + ] + }, + "from_id": { "$ref": "defs.schema.json#/$defs/entityId" }, + "to_id": { "$ref": "defs.schema.json#/$defs/entityId" }, + "confidence": { "$ref": "defs.schema.json#/$defs/confidence" }, + "source": { "$ref": "defs.schema.json#/$defs/sourceId" }, + "method": { + "enum": ["declared", "derived", "inferred", "manual"] + }, + "valid_from": { "$ref": "defs.schema.json#/$defs/nullableTimestamp" }, + "valid_to": { "$ref": "defs.schema.json#/$defs/nullableTimestamp" }, + "recorded_at": { "$ref": "defs.schema.json#/$defs/nullableTimestamp" }, + "superseded_at": { "$ref": "defs.schema.json#/$defs/nullableTimestamp" }, + "evidence": { + "type": "array", + "items": { + "type": "object", + "required": ["kind", "ref"], + "properties": { + "kind": { "enum": ["observation", "structural", "inference"] }, + "ref": { "type": "string" } + }, + "additionalProperties": false + } + }, + "properties": { "type": "object" }, + "notes": { "type": "string" } + }, + "if": { + "properties": { "method": { "const": "inferred" } }, + "required": ["method"] + }, + "then": { + "required": ["evidence"], + "properties": { + "evidence": { "type": "array", "minItems": 1 } + } + }, + "additionalProperties": false +} diff --git a/intelligence/schemas/sdui-panel.schema.json b/intelligence/schemas/sdui-panel.schema.json new file mode 100644 index 0000000..fc6d687 --- /dev/null +++ b/intelligence/schemas/sdui-panel.schema.json @@ -0,0 +1,57 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/sdui-panel.schema.json", + "title": "SDUI panel document", + "description": "Server-driven UI layout for an entity's intelligence view.", + "type": "object", + "required": ["entity_id", "entity_type", "situation", "layout"], + "properties": { + "entity_id": { "$ref": "defs.schema.json#/$defs/entityId" }, + "entity_type": { "$ref": "defs.schema.json#/$defs/entityType" }, + "situation": { + "type": "array", + "items": { + "enum": ["no-live-data", "stale-telemetry", "has-gaps", "estimated-load", "anomaly-active", "normal"] + } + }, + "layout": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["panel"], + "properties": { + "panel": { + "enum": [ + "headline-state", + "timeseries", + "gap-card", + "graph-neighborhood", + "inference-list", + "vessel-track", + "fuel-mix", + "price-strip", + "flow-arcs", + "table", + "markdown-note" + ] + }, + "props": { "type": "object" }, + "data": { + "type": "object", + "required": ["endpoint"], + "properties": { + "endpoint": { + "type": "string", + "pattern": "^/api/intel/" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false +} diff --git a/intelligence/schemas/source.schema.json b/intelligence/schemas/source.schema.json new file mode 100644 index 0000000..88d6f58 --- /dev/null +++ b/intelligence/schemas/source.schema.json @@ -0,0 +1,71 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/source.schema.json", + "title": "Source", + "description": "A registered data source (publisher + dataset) in the source registry.", + "type": "object", + "required": ["source_id", "publisher", "dataset", "api_type", "url", "coverage", "data_class", "licensing", "verification_status", "last_checked"], + "properties": { + "source_id": { "$ref": "defs.schema.json#/$defs/sourceId" }, + "publisher": { "type": "string" }, + "dataset": { "type": "string" }, + "api_type": { + "enum": ["rest", "websocket", "file-drop", "html", "bulk-download", "none"] + }, + "url": { "type": "string" }, + "docs_url": { "type": "string" }, + "coverage": { + "type": "object", + "required": ["grids", "countries"], + "properties": { + "grids": { + "type": "array", + "items": { "type": "string" } + }, + "countries": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": false + }, + "data_class": { + "enum": ["live", "static", "both"] + }, + "entity_types": { + "type": "array", + "items": { "type": "string" } + }, + "metrics": { + "type": "array", + "items": { "type": "string" } + }, + "latency": { "type": "string" }, + "resolution": { "type": "string" }, + "update_frequency": { "type": "string" }, + "history_depth": { "type": "string" }, + "licensing": { + "type": "object", + "required": ["class"], + "properties": { + "class": { + "enum": ["open", "open-attribution", "registration-required", "commercial", "restricted"] + }, + "notes": { "type": "string" } + }, + "additionalProperties": false + }, + "authentication": { + "enum": ["none", "api_key", "oauth", "registration"] + }, + "cost": { "type": "string" }, + "reliability": { "type": "integer", "minimum": 1, "maximum": 5 }, + "verification_status": { + "enum": ["verified_live", "documented_only", "unverified"] + }, + "last_checked": { "type": "string", "format": "date" }, + "quality_notes": { "type": "string" }, + "notes": { "type": "string" } + }, + "additionalProperties": false +} diff --git a/intelligence/schemas/telemetry-feed.schema.json b/intelligence/schemas/telemetry-feed.schema.json new file mode 100644 index 0000000..32d17d3 --- /dev/null +++ b/intelligence/schemas/telemetry-feed.schema.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pact.tailor.au/intelligence/schemas/telemetry-feed.schema.json", + "title": "Telemetry feed", + "description": "A concrete live-data feed binding a source to a connector and metrics.", + "type": "object", + "required": ["feed_id", "source_id", "connector_id", "metrics", "cadence"], + "properties": { + "feed_id": { "$ref": "defs.schema.json#/$defs/feedId" }, + "source_id": { "$ref": "defs.schema.json#/$defs/sourceId" }, + "connector_id": { "type": "string" }, + "metrics": { + "type": "array", + "items": { "$ref": "defs.schema.json#/$defs/metric" }, + "minItems": 1 + }, + "cadence": { "type": "string" }, + "description": { "type": "string" }, + "entity_scope": { "type": "string" }, + "notes": { "type": "string" } + }, + "additionalProperties": false +} From a06ad5ea5f7b3a603d9e0533ee81256372690121 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 01:37:27 +0000 Subject: [PATCH 03/16] =?UTF-8?q?feat(intelligence):=20store=20=E2=80=94?= =?UTF-8?q?=20bronze/silver/gold=20medallion=20(phase=206)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Embedded, zero-external-service storage under gitignored var/: immutable content-addressed bronze artifacts, append-only silver observation JSONL partitioned by feed and event date, and a hash-chained gold graph-delta journal (prev_hash over canonical JSON, GENESIS first — the PACT §6.4 construction) so runtime graph changes are tamper-evident and replayable. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014yVGqBtvM1zNAd9tvUVNzm --- intelligence/packages/store/package.json | 3 +- intelligence/packages/store/src/bronze.ts | 90 +++++++++++++ intelligence/packages/store/src/canonical.ts | 11 ++ intelligence/packages/store/src/gold.ts | 119 ++++++++++++++++++ intelligence/packages/store/src/index.ts | 26 +++- intelligence/packages/store/src/silver.ts | 51 ++++++++ .../packages/store/test/store.test.mjs | 93 ++++++++++++++ 7 files changed, 390 insertions(+), 3 deletions(-) create mode 100644 intelligence/packages/store/src/bronze.ts create mode 100644 intelligence/packages/store/src/canonical.ts create mode 100644 intelligence/packages/store/src/gold.ts create mode 100644 intelligence/packages/store/src/silver.ts create mode 100644 intelligence/packages/store/test/store.test.mjs diff --git a/intelligence/packages/store/package.json b/intelligence/packages/store/package.json index 4dc96d8..31fb8f1 100644 --- a/intelligence/packages/store/package.json +++ b/intelligence/packages/store/package.json @@ -13,7 +13,8 @@ "node": ">=20" }, "scripts": { - "build": "tsc -p tsconfig.json" + "build": "tsc -p tsconfig.json", + "test": "npm run build && node --test \"test/*.test.mjs\"" }, "dependencies": { "@pact-tailor/ontology": "*" diff --git a/intelligence/packages/store/src/bronze.ts b/intelligence/packages/store/src/bronze.ts new file mode 100644 index 0000000..d4c27ed --- /dev/null +++ b/intelligence/packages/store/src/bronze.ts @@ -0,0 +1,90 @@ +// Bronze: verbatim fetched bytes + fetch metadata. Immutable — a changed re-fetch is a +// new artifact (content-addressed suffix), never an overwrite. +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, readdirSync, writeFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; + +export interface BronzeMeta { + artifact_id: string; + source_id: string; + url: string; + fetched_at: string; + sha256: string; + http_status: number; + content_type?: string; + bytes: number; +} + +export interface BronzePut { + source_id: string; + url: string; + fetched_at: string; + http_status: number; + content_type?: string; + body: Buffer; +} + +export class BronzeStore { + constructor(private readonly root: string) {} + + private dirFor(sourceId: string, fetchedAt: string): string { + const day = fetchedAt.slice(0, 10); + const [yyyy = "0000", mm = "00", dd = "00"] = day.split("-"); + return join(this.root, sourceId.replaceAll(":", "_"), yyyy, mm, dd); + } + + put(input: BronzePut): BronzeMeta { + const sha256 = createHash("sha256").update(input.body).digest("hex"); + const stamp = input.fetched_at.replace(/[:.]/g, "-"); + const artifactId = `${stamp}-${sha256.slice(0, 12)}`; + const dir = this.dirFor(input.source_id, input.fetched_at); + mkdirSync(dir, { recursive: true }); + const binPath = join(dir, `${artifactId}.bin`); + const meta: BronzeMeta = { + artifact_id: artifactId, + source_id: input.source_id, + url: input.url, + fetched_at: input.fetched_at, + sha256, + http_status: input.http_status, + ...(input.content_type !== undefined ? { content_type: input.content_type } : {}), + bytes: input.body.length, + }; + if (!existsSync(binPath)) { + writeFileSync(binPath, input.body); + writeFileSync(join(dir, `${artifactId}.meta.json`), JSON.stringify(meta, null, 2)); + } + return meta; + } + + get(sourceId: string, artifactId: string): { meta: BronzeMeta; body: Buffer } | null { + // Artifact IDs embed their fetch timestamp, which locates the directory. + const fetchedAt = artifactId.slice(0, 10); + const dir = this.dirFor(sourceId, fetchedAt); + const binPath = join(dir, `${artifactId}.bin`); + if (!existsSync(binPath)) return null; + const meta = JSON.parse(readFileSync(join(dir, `${artifactId}.meta.json`), "utf8")) as BronzeMeta; + return { meta, body: readFileSync(binPath) }; + } + + list(sourceId: string): BronzeMeta[] { + const base = join(this.root, sourceId.replaceAll(":", "_")); + const out: BronzeMeta[] = []; + const walk = (dir: string) => { + let names: string[]; + try { + names = readdirSync(dir, { withFileTypes: true }).map((d) => (d.isDirectory() ? `${d.name}/` : d.name)); + } catch { + return; + } + for (const name of names.sort()) { + if (name.endsWith("/")) walk(join(dir, name.slice(0, -1))); + else if (name.endsWith(".meta.json")) { + out.push(JSON.parse(readFileSync(join(dir, name), "utf8")) as BronzeMeta); + } + } + }; + walk(base); + return out; + } +} diff --git a/intelligence/packages/store/src/canonical.ts b/intelligence/packages/store/src/canonical.ts new file mode 100644 index 0000000..98d6a2a --- /dev/null +++ b/intelligence/packages/store/src/canonical.ts @@ -0,0 +1,11 @@ +// Deterministic JSON: recursively key-sorted, no whitespace. Same construction the +// PACT event log uses for its hash chain (RFC 8785 in spirit; sufficient here because +// we only hash JSON we ourselves serialised). +export function canonicalJson(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + const entries = Object.entries(value as Record) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`).join(",")}}`; +} diff --git a/intelligence/packages/store/src/gold.ts b/intelligence/packages/store/src/gold.ts new file mode 100644 index 0000000..2905f30 --- /dev/null +++ b/intelligence/packages/store/src/gold.ts @@ -0,0 +1,119 @@ +// Gold: derived, always reproducible from bronze + code. The graph delta journal is the +// load-bearing piece — hash-chained exactly like the PACT §6.4 event log (prev_hash over +// the canonical JSON of the previous line, "GENESIS" first), so runtime graph changes +// are tamper-evident and replayable. Memory is a projection; this file is the truth. +import { appendFileSync, mkdirSync, readFileSync, readdirSync, writeFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import { createHash } from "node:crypto"; +import type { InferenceRecord } from "@pact-tailor/ontology"; +import { canonicalJson } from "./canonical.js"; + +export interface GraphDelta { + seq: number; + recorded_at: string; + kind: "entity_patch" | "relationship_patch"; + payload: Record; + prev_hash: string; +} + +export type GraphDeltaInput = Omit; + +function hashLine(line: string): string { + return createHash("sha256").update(line).digest("base64url"); +} + +export class GoldStore { + private readonly graphDir: string; + private readonly stateDir: string; + private readonly inferDir: string; + private lastHash: string | null = null; + private lastSeq = 0; + + constructor(root: string) { + this.graphDir = join(root, "graph"); + this.stateDir = join(root, "state"); + this.inferDir = join(root, "inferences"); + } + + private get deltasPath(): string { + return join(this.graphDir, "deltas.jsonl"); + } + + private loadChainTip(): void { + if (this.lastHash !== null) return; + if (!existsSync(this.deltasPath)) { + this.lastHash = "GENESIS"; + return; + } + const lines = readFileSync(this.deltasPath, "utf8").split("\n").filter((l) => l.trim().length > 0); + const last = lines[lines.length - 1]; + if (last === undefined) { + this.lastHash = "GENESIS"; + return; + } + const parsed = JSON.parse(last) as GraphDelta; + this.lastSeq = parsed.seq; + this.lastHash = hashLine(canonicalJson(parsed)); + } + + appendGraphDelta(input: GraphDeltaInput): GraphDelta { + this.loadChainTip(); + const delta: GraphDelta = { + seq: this.lastSeq + 1, + recorded_at: input.recorded_at, + kind: input.kind, + payload: input.payload, + prev_hash: this.lastHash as string, + }; + mkdirSync(this.graphDir, { recursive: true }); + const line = canonicalJson(delta); + appendFileSync(this.deltasPath, line + "\n"); + this.lastSeq = delta.seq; + this.lastHash = hashLine(line); + return delta; + } + + readGraphDeltas(): GraphDelta[] { + if (!existsSync(this.deltasPath)) return []; + return readFileSync(this.deltasPath, "utf8") + .split("\n") + .filter((l) => l.trim().length > 0) + .map((l) => JSON.parse(l) as GraphDelta); + } + + /** Walk the journal verifying every prev_hash. Returns the first bad seq, or null if intact. */ + verifyChain(): number | null { + let prev = "GENESIS"; + for (const delta of this.readGraphDeltas()) { + if (delta.prev_hash !== prev) return delta.seq; + prev = hashLine(canonicalJson(delta)); + } + return null; + } + + snapshotState(snapshot: Record, at: string): string { + mkdirSync(this.stateDir, { recursive: true }); + const path = join(this.stateDir, `${at.replace(/[:.]/g, "-")}.json`); + writeFileSync(path, JSON.stringify(snapshot, null, 2)); + return path; + } + + appendInference(inference: InferenceRecord): void { + mkdirSync(this.inferDir, { recursive: true }); + const day = inference.produced_at.slice(0, 10); + appendFileSync(join(this.inferDir, `${day}.jsonl`), JSON.stringify(inference) + "\n"); + } + + readInferences(): InferenceRecord[] { + if (!existsSync(this.inferDir)) return []; + return readdirSync(this.inferDir) + .sort() + .filter((n) => n.endsWith(".jsonl")) + .flatMap((n) => + readFileSync(join(this.inferDir, n), "utf8") + .split("\n") + .filter((l) => l.trim().length > 0) + .map((l) => JSON.parse(l) as InferenceRecord), + ); + } +} diff --git a/intelligence/packages/store/src/index.ts b/intelligence/packages/store/src/index.ts index c0d2c2a..2be9482 100644 --- a/intelligence/packages/store/src/index.ts +++ b/intelligence/packages/store/src/index.ts @@ -1,2 +1,24 @@ -// @pact-tailor/store — medallion storage. Populated in the store commit. -export {}; +// @pact-tailor/store — the embedded medallion. Layout contract in docs/STORAGE.md: +// bronze is immutable, silver is append-only, gold is disposable-and-reproducible. +// Nothing outside this package touches `var/` directly. +import { join } from "node:path"; +import { BronzeStore } from "./bronze.js"; +import { SilverStore } from "./silver.js"; +import { GoldStore } from "./gold.js"; + +export { BronzeStore, type BronzeMeta, type BronzePut } from "./bronze.js"; +export { SilverStore } from "./silver.js"; +export { GoldStore, type GraphDelta, type GraphDeltaInput } from "./gold.js"; +export { canonicalJson } from "./canonical.js"; + +export class MedallionStore { + readonly bronze: BronzeStore; + readonly silver: SilverStore; + readonly gold: GoldStore; + + constructor(varDir: string) { + this.bronze = new BronzeStore(join(varDir, "bronze")); + this.silver = new SilverStore(join(varDir, "silver")); + this.gold = new GoldStore(join(varDir, "gold")); + } +} diff --git a/intelligence/packages/store/src/silver.ts b/intelligence/packages/store/src/silver.ts new file mode 100644 index 0000000..35a9f1b --- /dev/null +++ b/intelligence/packages/store/src/silver.ts @@ -0,0 +1,51 @@ +// Silver: normalised Observation envelopes, append-only JSONL, partitioned by feed and +// event date. Corrections are new lines (is_correction: true); nothing is rewritten. +import { appendFileSync, mkdirSync, readFileSync, readdirSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import type { ObservationRecord } from "@pact-tailor/ontology"; + +export class SilverStore { + constructor(private readonly root: string) {} + + private feedDir(feedId: string): string { + return join(this.root, feedId.replaceAll(":", "_")); + } + + append(observations: ObservationRecord[]): void { + const byFile = new Map(); + for (const obs of observations) { + const day = obs.event_time.slice(0, 10); + const file = join(this.feedDir(obs.feed_id), `${day}.jsonl`); + let lines = byFile.get(file); + if (!lines) byFile.set(file, (lines = [])); + lines.push(JSON.stringify(obs)); + } + for (const [file, lines] of byFile) { + mkdirSync(join(file, ".."), { recursive: true }); + appendFileSync(file, lines.join("\n") + "\n"); + } + } + + readFeed(feedId: string, options: { fromDay?: string; toDay?: string } = {}): ObservationRecord[] { + const dir = this.feedDir(feedId); + if (!existsSync(dir)) return []; + const out: ObservationRecord[] = []; + for (const name of readdirSync(dir).sort()) { + if (!name.endsWith(".jsonl")) continue; + const day = name.slice(0, 10); + if (options.fromDay && day < options.fromDay) continue; + if (options.toDay && day > options.toDay) continue; + for (const line of readFileSync(join(dir, name), "utf8").split("\n")) { + if (line.trim().length > 0) out.push(JSON.parse(line) as ObservationRecord); + } + } + return out; + } + + listFeeds(): string[] { + if (!existsSync(this.root)) return []; + return readdirSync(this.root) + .sort() + .map((name) => name.replaceAll("_", ":")); + } +} diff --git a/intelligence/packages/store/test/store.test.mjs b/intelligence/packages/store/test/store.test.mjs new file mode 100644 index 0000000..6d4c800 --- /dev/null +++ b/intelligence/packages/store/test/store.test.mjs @@ -0,0 +1,93 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { MedallionStore, canonicalJson } from "../dist/index.js"; + +const obs = (feed, day, hour, value, extra = {}) => ({ + observation_id: "01JABCDEFGHJKMNPQRSTVWXYZ0".slice(0, 26), + entity_id: "gen:test:alpha", + metric: "power.output.mw", + value, + unit: "MW", + event_time: `${day}T${hour}:00:00Z`, + ingest_time: `${day}T${hour}:01:00Z`, + source_id: "source:test:feed", + feed_id: feed, + quality: "good", + ...extra, +}); + +test("bronze: put is content-addressed, immutable, and re-readable", () => { + const store = new MedallionStore(mkdtempSync(join(tmpdir(), "pt-store-"))); + const body = Buffer.from("PUBLIC_DISPATCHIS payload"); + const meta = store.bronze.put({ + source_id: "source:aemo:nemweb-dispatchis", + url: "https://example.invalid/PUBLIC_DISPATCHIS.zip", + fetched_at: "2026-08-12T01:05:00.000Z", + http_status: 200, + body, + }); + assert.equal(meta.bytes, body.length); + const roundTrip = store.bronze.get("source:aemo:nemweb-dispatchis", meta.artifact_id); + assert.ok(roundTrip); + assert.deepEqual(roundTrip.body, body); + // Same content re-put → same artifact, still listed once. + store.bronze.put({ + source_id: "source:aemo:nemweb-dispatchis", + url: "https://example.invalid/PUBLIC_DISPATCHIS.zip", + fetched_at: "2026-08-12T01:05:00.000Z", + http_status: 200, + body, + }); + assert.equal(store.bronze.list("source:aemo:nemweb-dispatchis").length, 1); +}); + +test("silver: append partitions by feed and event date; reads filter by day", () => { + const store = new MedallionStore(mkdtempSync(join(tmpdir(), "pt-store-"))); + store.silver.append([ + obs("feed:test:a", "2026-08-11", "23", 100), + obs("feed:test:a", "2026-08-12", "00", 110), + obs("feed:test:b", "2026-08-12", "00", 7), + ]); + assert.equal(store.silver.readFeed("feed:test:a").length, 2); + assert.equal(store.silver.readFeed("feed:test:a", { fromDay: "2026-08-12" }).length, 1); + assert.deepEqual(store.silver.listFeeds().sort(), ["feed:test:a", "feed:test:b"]); +}); + +test("gold: graph delta journal is hash-chained and tamper-evident", () => { + const dir = mkdtempSync(join(tmpdir(), "pt-store-")); + const store = new MedallionStore(dir); + for (let i = 1; i <= 3; i++) { + store.gold.appendGraphDelta({ + recorded_at: `2026-08-12T0${i}:00:00Z`, + kind: "relationship_patch", + payload: { rel_id: `rel:test-${i}` }, + }); + } + assert.equal(store.gold.verifyChain(), null); + const deltas = store.gold.readGraphDeltas(); + assert.deepEqual(deltas.map((d) => d.seq), [1, 2, 3]); + assert.equal(deltas[0].prev_hash, "GENESIS"); + + // A fresh instance resumes the chain from disk. + const reopened = new MedallionStore(dir); + reopened.gold.appendGraphDelta({ + recorded_at: "2026-08-12T04:00:00Z", + kind: "entity_patch", + payload: { entity_id: "gen:test:alpha" }, + }); + assert.equal(reopened.gold.verifyChain(), null); + + // Tampering with a middle line breaks verification at the next seq. + const path = join(dir, "gold", "graph", "deltas.jsonl"); + const lines = readFileSync(path, "utf8").trim().split("\n"); + const tampered = lines[1].replace("rel:test-2", "rel:evil"); + writeFileSync(path, [lines[0], tampered, ...lines.slice(2)].join("\n") + "\n"); + assert.notEqual(new MedallionStore(dir).gold.verifyChain(), null); +}); + +test("canonicalJson: key order does not change the serialisation", () => { + assert.equal(canonicalJson({ b: 1, a: { d: 2, c: 3 } }), canonicalJson({ a: { c: 3, d: 2 }, b: 1 })); +}); From 72c1c855fee428188258b17b2321291731f6e63f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 01:37:27 +0000 Subject: [PATCH 04/16] feat(intelligence): bitemporal knowledge graph engine (phases 5, 15) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In-process property graph over canonical entities: world-time and system-time filters on every read, upstream/downstream supply-chain traversal, explicit rel-type chains (mine → terminal → port), re-record supersession that keeps prior belief queryable, deterministic export. Zero runtime deps. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014yVGqBtvM1zNAd9tvUVNzm --- intelligence/packages/graph/package.json | 3 +- intelligence/packages/graph/src/index.ts | 202 +++++++++++++++++- .../packages/graph/test/graph.test.mjs | 128 +++++++++++ 3 files changed, 330 insertions(+), 3 deletions(-) create mode 100644 intelligence/packages/graph/test/graph.test.mjs diff --git a/intelligence/packages/graph/package.json b/intelligence/packages/graph/package.json index 58af249..c0a888e 100644 --- a/intelligence/packages/graph/package.json +++ b/intelligence/packages/graph/package.json @@ -13,7 +13,8 @@ "node": ">=20" }, "scripts": { - "build": "tsc -p tsconfig.json" + "build": "tsc -p tsconfig.json", + "test": "npm run build && node --test \"test/*.test.mjs\"" }, "dependencies": { "@pact-tailor/ontology": "*" diff --git a/intelligence/packages/graph/src/index.ts b/intelligence/packages/graph/src/index.ts index 91ee4fb..e79ca3a 100644 --- a/intelligence/packages/graph/src/index.ts +++ b/intelligence/packages/graph/src/index.ts @@ -1,2 +1,200 @@ -// @pact-tailor/graph — bitemporal property graph. Populated in the graph commit. -export {}; +// @pact-tailor/graph — in-process bitemporal property graph over the structural world +// model. Nodes are canonical entities (never mutated by telemetry); edges carry the full +// relationship envelope with two time axes. Runtime changes enter only via applyDelta, +// which the caller journals through @pact-tailor/store's hash-chained gold log first — +// the journal is truth, this structure is the projection. +import type { EntityRecord, RelationshipRecord, RelType } from "@pact-tailor/ontology"; + +export interface TemporalFilter { + /** World-time: the edge must be valid at this instant (default: now). */ + asOf?: string; + /** System-time: only edges we believed at this instant (default: latest). */ + recordedAsOf?: string; +} + +export interface NeighborOptions extends TemporalFilter { + relTypes?: RelType[]; + direction?: "out" | "in" | "both"; +} + +export interface TraverseOptions extends TemporalFilter { + start: string; + direction: "upstream" | "downstream"; + relTypes?: RelType[]; + maxDepth?: number; +} + +export interface PathStep { + rel: RelationshipRecord; + from: string; + to: string; +} + +export interface Path { + nodes: string[]; + steps: PathStep[]; +} + +function activeAt(rel: RelationshipRecord, filter: TemporalFilter): boolean { + const asOf = filter.asOf ?? new Date().toISOString(); + if (rel.valid_from != null && rel.valid_from > asOf) return false; + if (rel.valid_to != null && rel.valid_to <= asOf) return false; + if (filter.recordedAsOf !== undefined) { + if (rel.recorded_at != null && rel.recorded_at > filter.recordedAsOf) return false; + if (rel.superseded_at != null && rel.superseded_at <= filter.recordedAsOf) return false; + } else if (rel.superseded_at != null) { + return false; + } + return true; +} + +export class KnowledgeGraph { + private nodes = new Map(); + private edges = new Map(); + private out = new Map(); + private in = new Map(); + + load(entities: EntityRecord[], relationships: RelationshipRecord[]): void { + for (const entity of entities) this.nodes.set(entity.entity_id, entity); + for (const rel of relationships) this.indexEdge(rel); + } + + private indexEdge(rel: RelationshipRecord): void { + this.edges.set(rel.rel_id, rel); + let outList = this.out.get(rel.from_id); + if (!outList) this.out.set(rel.from_id, (outList = [])); + outList.push(rel); + let inList = this.in.get(rel.to_id); + if (!inList) this.in.set(rel.to_id, (inList = [])); + inList.push(rel); + } + + /** + * Apply a runtime patch. The caller MUST have journaled it (GoldStore.appendGraphDelta) + * before calling — this method only projects. + */ + applyDelta(delta: + | { kind: "entity_patch"; entity: EntityRecord } + | { kind: "relationship_patch"; relationship: RelationshipRecord }): void { + if (delta.kind === "entity_patch") { + this.nodes.set(delta.entity.entity_id, delta.entity); + return; + } + const rel = delta.relationship; + const existing = this.edges.get(rel.rel_id); + if (existing) { + // Re-recording an edge supersedes the old version rather than replacing history. + const closed: RelationshipRecord = { ...existing, superseded_at: rel.recorded_at ?? new Date().toISOString() }; + this.edges.set(existing.rel_id, closed); + const replaceIn = (list: RelationshipRecord[] | undefined) => { + if (!list) return; + const idx = list.indexOf(existing); + if (idx >= 0) list[idx] = closed; + }; + replaceIn(this.out.get(existing.from_id)); + replaceIn(this.in.get(existing.to_id)); + this.indexEdge({ ...rel, rel_id: `${rel.rel_id}-r${Date.now()}` }); + return; + } + this.indexEdge(rel); + } + + node(id: string): EntityRecord | null { + return this.nodes.get(id) ?? null; + } + + nodeCount(): number { + return this.nodes.size; + } + + edgeCount(): number { + return this.edges.size; + } + + allNodes(): EntityRecord[] { + return [...this.nodes.values()]; + } + + neighbors(id: string, options: NeighborOptions = {}): PathStep[] { + const direction = options.direction ?? "both"; + const steps: PathStep[] = []; + const want = options.relTypes ? new Set(options.relTypes) : null; + if (direction !== "in") { + for (const rel of this.out.get(id) ?? []) { + if (want && !want.has(rel.rel_type)) continue; + if (!activeAt(rel, options)) continue; + steps.push({ rel, from: id, to: rel.to_id }); + } + } + if (direction !== "out") { + for (const rel of this.in.get(id) ?? []) { + if (want && !want.has(rel.rel_type)) continue; + if (!activeAt(rel, options)) continue; + steps.push({ rel, from: rel.from_id, to: id }); + } + } + return steps; + } + + /** + * Breadth-first supply-chain walk. "downstream" follows edges in their natural + * direction (mine SUPPLIES station, smelter SHIPS_TO market...); "upstream" walks + * them in reverse (what feeds this entity?). + */ + traverse(options: TraverseOptions): Path[] { + const maxDepth = options.maxDepth ?? 4; + const paths: Path[] = []; + const queue: Path[] = [{ nodes: [options.start], steps: [] }]; + const visited = new Set([options.start]); + + while (queue.length > 0) { + const path = queue.shift() as Path; + if (path.steps.length >= maxDepth) continue; + const tip = path.nodes[path.nodes.length - 1] as string; + const direction = options.direction === "downstream" ? "out" : "in"; + const steps = this.neighbors(tip, { + direction, + ...(options.relTypes !== undefined ? { relTypes: options.relTypes } : {}), + ...(options.asOf !== undefined ? { asOf: options.asOf } : {}), + ...(options.recordedAsOf !== undefined ? { recordedAsOf: options.recordedAsOf } : {}), + }); + for (const step of steps) { + const next = options.direction === "downstream" ? step.to : step.from; + if (visited.has(next)) continue; + visited.add(next); + const extended: Path = { nodes: [...path.nodes, next], steps: [...path.steps, step] }; + paths.push(extended); + queue.push(extended); + } + } + return paths; + } + + /** + * Follow an explicit relationship-type chain from a start node, e.g. + * chain("mine:au-qld:blackwater", ["TRANSPORTS", "PART_OF"]) — returns every path + * that matches the chain exactly, in order. + */ + chain(start: string, relTypes: RelType[], filter: TemporalFilter = {}): Path[] { + let frontier: Path[] = [{ nodes: [start], steps: [] }]; + for (const relType of relTypes) { + const next: Path[] = []; + for (const path of frontier) { + const tip = path.nodes[path.nodes.length - 1] as string; + for (const step of this.neighbors(tip, { direction: "out", relTypes: [relType], ...filter })) { + next.push({ nodes: [...path.nodes, step.to], steps: [...path.steps, step] }); + } + } + frontier = next; + if (frontier.length === 0) break; + } + return frontier; + } + + /** Deterministic export for tests and the globe's static bundle. */ + export(): { entities: EntityRecord[]; relationships: RelationshipRecord[] } { + const entities = [...this.nodes.values()].sort((a, b) => a.entity_id.localeCompare(b.entity_id)); + const relationships = [...this.edges.values()].sort((a, b) => a.rel_id.localeCompare(b.rel_id)); + return { entities, relationships }; + } +} diff --git a/intelligence/packages/graph/test/graph.test.mjs b/intelligence/packages/graph/test/graph.test.mjs new file mode 100644 index 0000000..34486d0 --- /dev/null +++ b/intelligence/packages/graph/test/graph.test.mjs @@ -0,0 +1,128 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { KnowledgeGraph } from "../dist/index.js"; + +const entity = (id, type, name) => ({ + entity_id: id, + entity_type: type, + name, + data_class: "structural", + observability: "UNCLASSIFIED", + sources: ["source:test:feed"], +}); + +const rel = (id, type, from, to, extra = {}) => ({ + rel_id: id, + rel_type: type, + from_id: from, + to_id: to, + confidence: 1.0, + source: "source:test:feed", + method: "declared", + ...extra, +}); + +function gladstoneGraph() { + const g = new KnowledgeGraph(); + g.load( + [ + entity("mine:au-qld:blackwater", "CoalMine", "Blackwater"), + entity("terminal:au-qld:rg-tanna", "Terminal", "RG Tanna"), + entity("port:au-qld:gladstone", "Port", "Port of Gladstone"), + entity("vessel:imo:9000001", "Vessel", "MV Bowen Trader"), + entity("dest:jp", "Destination", "Japan"), + entity("gen:au-nem:gladstone-ps", "Generator", "Gladstone PS"), + entity("smelter:au-qld:boyne-island", "Smelter", "Boyne Island"), + ], + [ + rel("rel-1", "TRANSPORTS", "mine:au-qld:blackwater", "terminal:au-qld:rg-tanna"), + rel("rel-2", "PART_OF", "terminal:au-qld:rg-tanna", "port:au-qld:gladstone"), + rel("rel-3", "LOADS_AT", "vessel:imo:9000001", "terminal:au-qld:rg-tanna"), + rel("rel-4", "SHIPS_TO", "vessel:imo:9000001", "dest:jp"), + rel("rel-5", "SUPPLIES", "gen:au-nem:gladstone-ps", "smelter:au-qld:boyne-island"), + ], + ); + return g; +} + +test("neighbors respects direction and relTypes", () => { + const g = gladstoneGraph(); + const out = g.neighbors("mine:au-qld:blackwater", { direction: "out" }); + assert.equal(out.length, 1); + assert.equal(out[0].to, "terminal:au-qld:rg-tanna"); + const loaders = g.neighbors("terminal:au-qld:rg-tanna", { direction: "in", relTypes: ["LOADS_AT"] }); + assert.equal(loaders.length, 1); + assert.equal(loaders[0].from, "vessel:imo:9000001"); +}); + +test("downstream traversal walks mine → terminal → port", () => { + const g = gladstoneGraph(); + const paths = g.traverse({ start: "mine:au-qld:blackwater", direction: "downstream", maxDepth: 3 }); + const tips = paths.map((p) => p.nodes[p.nodes.length - 1]); + assert.ok(tips.includes("terminal:au-qld:rg-tanna")); + assert.ok(tips.includes("port:au-qld:gladstone")); +}); + +test("upstream traversal answers 'what feeds the smelter?'", () => { + const g = gladstoneGraph(); + const paths = g.traverse({ start: "smelter:au-qld:boyne-island", direction: "upstream", maxDepth: 2 }); + assert.ok(paths.some((p) => p.nodes.includes("gen:au-nem:gladstone-ps"))); +}); + +test("chain follows an explicit rel-type sequence", () => { + const g = gladstoneGraph(); + const paths = g.chain("mine:au-qld:blackwater", ["TRANSPORTS", "PART_OF"]); + assert.equal(paths.length, 1); + assert.deepEqual(paths[0].nodes, ["mine:au-qld:blackwater", "terminal:au-qld:rg-tanna", "port:au-qld:gladstone"]); +}); + +test("bitemporal: expired edges disappear under asOf; superseded edges disappear by default", () => { + const g = new KnowledgeGraph(); + g.load( + [entity("mine:t:a", "Mine", "A"), entity("gen:t:b", "Generator", "B")], + [ + rel("rel-old", "SUPPLIES", "mine:t:a", "gen:t:b", { + valid_from: "2010-01-01T00:00:00Z", + valid_to: "2019-12-31T00:00:00Z", + }), + ], + ); + assert.equal(g.neighbors("mine:t:a", { direction: "out" }).length, 0); + assert.equal(g.neighbors("mine:t:a", { direction: "out", asOf: "2015-06-01T00:00:00Z" }).length, 1); + + g.applyDelta({ + kind: "relationship_patch", + relationship: rel("rel-new", "SUPPLIES", "mine:t:a", "gen:t:b", { + valid_from: "2020-01-01T00:00:00Z", + recorded_at: "2020-01-05T00:00:00Z", + }), + }); + const now = g.neighbors("mine:t:a", { direction: "out" }); + assert.equal(now.length, 1); + assert.equal(now[0].rel.rel_id, "rel-new"); +}); + +test("applyDelta supersedes a re-recorded edge without losing history", () => { + const g = new KnowledgeGraph(); + g.load( + [entity("mine:t:a", "Mine", "A"), entity("gen:t:b", "Generator", "B")], + [rel("rel-x", "SUPPLIES", "mine:t:a", "gen:t:b", { confidence: 0.6, recorded_at: "2026-01-01T00:00:00Z" })], + ); + g.applyDelta({ + kind: "relationship_patch", + relationship: rel("rel-x", "SUPPLIES", "mine:t:a", "gen:t:b", { confidence: 0.95, recorded_at: "2026-06-01T00:00:00Z" }), + }); + const current = g.neighbors("mine:t:a", { direction: "out" }); + assert.equal(current.length, 1); + assert.equal(current[0].rel.confidence, 0.95); + // System-time query as of before the re-record still sees the old belief. + const then = g.neighbors("mine:t:a", { direction: "out", recordedAsOf: "2026-03-01T00:00:00Z" }); + assert.equal(then.length, 1); + assert.equal(then[0].rel.confidence, 0.6); +}); + +test("export is deterministic", () => { + const a = gladstoneGraph().export(); + const b = gladstoneGraph().export(); + assert.deepEqual(a, b); +}); From 3f14a9eaaa7ac5a3ea6953e1494f04c8421b1c6d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 01:37:27 +0000 Subject: [PATCH 05/16] feat(intelligence): registry loader + cross-reference validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit YAML registry loading with per-record schema validation, referential integrity across grids/sources/gaps/entities/relationships (every cited ID must resolve), Grid-entity materialisation, and entity-resolution candidates (exact external-ID and name+proximity passes — reported, never auto-merged). validate:data CLI wired for CI. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014yVGqBtvM1zNAd9tvUVNzm --- intelligence/packages/registry/package.json | 3 +- .../registry/scripts/validate-data.mjs | 49 +++++ intelligence/packages/registry/src/index.ts | 187 +++++++++++++++++- intelligence/packages/registry/src/resolve.ts | 99 ++++++++++ .../packages/registry/test/registry.test.mjs | 159 +++++++++++++++ 5 files changed, 494 insertions(+), 3 deletions(-) create mode 100644 intelligence/packages/registry/scripts/validate-data.mjs create mode 100644 intelligence/packages/registry/src/resolve.ts create mode 100644 intelligence/packages/registry/test/registry.test.mjs diff --git a/intelligence/packages/registry/package.json b/intelligence/packages/registry/package.json index 2227fa8..c4cfb1d 100644 --- a/intelligence/packages/registry/package.json +++ b/intelligence/packages/registry/package.json @@ -13,7 +13,8 @@ "node": ">=20" }, "scripts": { - "build": "tsc -p tsconfig.json" + "build": "tsc -p tsconfig.json", + "test": "npm run build && node --test \"test/*.test.mjs\"" }, "dependencies": { "@pact-tailor/ontology": "*", diff --git a/intelligence/packages/registry/scripts/validate-data.mjs b/intelligence/packages/registry/scripts/validate-data.mjs new file mode 100644 index 0000000..68e1856 --- /dev/null +++ b/intelligence/packages/registry/scripts/validate-data.mjs @@ -0,0 +1,49 @@ +#!/usr/bin/env node +// Validates every YAML record under intelligence/data/ against its schema, then runs +// cross-registry referential integrity and entity-resolution sanity. CI-facing: +// exits non-zero on any problem. Run from anywhere: paths resolve relative to this file. +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; +import { loadRegistry, checkIntegrity, gridEntitiesFrom, findResolutionCandidates } from "@pact-tailor/registry"; + +const here = dirname(fileURLToPath(import.meta.url)); +const dataDir = join(here, "..", "..", "..", "data"); + +const { data, errors } = loadRegistry(dataDir); +const integrity = checkIntegrity(data); +const candidates = findResolutionCandidates([...data.entities, ...gridEntitiesFrom(data.grids)]).filter( + // Shared-external-ID hits in curated data are duplicate bugs; heuristic name hits are review items. + (c) => c.score === 1, +); + +const counts = { + grids: data.grids.length, + sources: data.sources.length, + gaps: data.gaps.length, + entities: data.entities.length, + relationships: data.relationships.length, +}; +console.log(`registry: ${JSON.stringify(counts)}`); + +let failed = false; +if (errors.length > 0) { + failed = true; + console.error(`\nSchema validation errors (${errors.length}):`); + for (const e of errors) console.error(` - ${e}`); +} +if (integrity.length > 0) { + failed = true; + console.error(`\nReferential integrity errors (${integrity.length}):`); + for (const e of integrity) console.error(` - ${e}`); +} +if (candidates.length > 0) { + failed = true; + console.error(`\nDuplicate-entity candidates via shared external IDs (${candidates.length}):`); + for (const c of candidates) console.error(` - ${c.a} ~ ${c.b} (${c.reason})`); +} + +if (failed) { + console.error("\nvalidate:data FAILED"); + process.exit(1); +} +console.log("validate:data OK"); diff --git a/intelligence/packages/registry/src/index.ts b/intelligence/packages/registry/src/index.ts index fb2cb6e..7fb6251 100644 --- a/intelligence/packages/registry/src/index.ts +++ b/intelligence/packages/registry/src/index.ts @@ -1,2 +1,185 @@ -// @pact-tailor/registry — registry loading and validation. Populated in the registry commit. -export {}; +// @pact-tailor/registry — loads the YAML registries (class-A structural truth) and +// enforces referential integrity across them. The registries on disk are the source +// of truth; this package never mutates them. +import { readFileSync, readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { parse } from "yaml"; +import { + createValidators, + type EntityRecord, + type GridRecord, + type ObservabilityGapRecord, + type RelationshipRecord, + type SourceRecord, + type Validators, +} from "@pact-tailor/ontology"; + +export { findResolutionCandidates, type ResolutionCandidate } from "./resolve.js"; + +export interface RegistryData { + grids: GridRecord[]; + sources: SourceRecord[]; + gaps: ObservabilityGapRecord[]; + entities: EntityRecord[]; + relationships: RelationshipRecord[]; +} + +export interface LoadResult { + data: RegistryData; + /** Validation errors, each prefixed with the offending file path. */ + errors: string[]; +} + +function yamlFilesUnder(dir: string): string[] { + let names: string[]; + try { + names = readdirSync(dir); + } catch { + return []; + } + const files: string[] = []; + for (const name of names.sort()) { + const full = join(dir, name); + if (statSync(full).isDirectory()) files.push(...yamlFilesUnder(full)); + else if (name.endsWith(".yaml") || name.endsWith(".yml")) files.push(full); + } + return files; +} + +function readRecords(file: string, errors: string[]): unknown[] { + let doc: unknown; + try { + doc = parse(readFileSync(file, "utf8")); + } catch (err) { + errors.push(`${file}: YAML parse error — ${(err as Error).message}`); + return []; + } + if (doc === null || typeof doc !== "object" || !Array.isArray((doc as { records?: unknown }).records)) { + errors.push(`${file}: expected a top-level \`records:\` array`); + return []; + } + return (doc as { records: unknown[] }).records; +} + +function loadKind( + dir: string, + validate: (data: unknown) => { ok: true; value: T } | { ok: false; errors: string[] }, + label: string, + errors: string[], +): T[] { + const out: T[] = []; + for (const file of yamlFilesUnder(dir)) { + const records = readRecords(file, errors); + records.forEach((record, i) => { + const result = validate(record); + if (result.ok) out.push(result.value); + else errors.push(`${file} ${label}[${i}]: ${result.errors.join("; ")}`); + }); + } + return out; +} + +/** + * Load every registry under `dataDir` (normally `intelligence/data`), validating each + * record against its schema. Schema-invalid records are excluded from the result and + * reported in `errors`. + */ +export function loadRegistry(dataDir: string, validators: Validators = createValidators()): LoadResult { + const errors: string[] = []; + const data: RegistryData = { + grids: loadKind(join(dataDir, "registries", "grids"), validators.grid, "grid", errors), + sources: loadKind(join(dataDir, "registries", "sources"), validators.source, "source", errors), + gaps: loadKind(join(dataDir, "registries", "gaps"), validators.gap, "gap", errors), + entities: loadKind(join(dataDir, "entities"), validators.entity, "entity", errors), + relationships: loadKind(join(dataDir, "relationships"), validators.relationship, "relationship", errors), + }; + return { data, errors }; +} + +const AVAILABILITY_TO_OBSERVABILITY: Record = { + rich: "KNOWN_LIVE", + partial: "ESTIMATED", + minimal: "ESTIMATED", + none: "NOT_OBSERVABLE", +}; + +/** Materialise Grid entities from grid registry records so grids are graph-addressable. */ +export function gridEntitiesFrom(grids: GridRecord[]): EntityRecord[] { + return grids.map((grid) => ({ + entity_id: grid.grid_id, + entity_type: "Grid", + name: grid.name, + data_class: "structural", + observability: AVAILABILITY_TO_OBSERVABILITY[grid.live_data.availability] ?? "UNCLASSIFIED", + geometry: null, + country: grid.countries[0] ?? null, + grid_id: grid.parent_grid ?? null, + properties: { kind: grid.kind, frequency_hz: grid.frequency_hz }, + external_ids: {}, + sources: grid.sources, + ...(grid.notes !== undefined ? { notes: grid.notes } : {}), + })); +} + +function duplicates(ids: string[]): string[] { + const seen = new Set(); + const dups = new Set(); + for (const id of ids) { + if (seen.has(id)) dups.add(id); + seen.add(id); + } + return [...dups]; +} + +/** + * Cross-registry referential integrity. Every ID mentioned anywhere must resolve: + * source citations, gap targets, relationship endpoints, grid parents, same_as. + */ +export function checkIntegrity(data: RegistryData): string[] { + const problems: string[] = []; + const sourceIds = new Set(data.sources.map((s) => s.source_id)); + const gridIds = new Set(data.grids.map((g) => g.grid_id)); + const entityIds = new Set(data.entities.map((e) => e.entity_id)); + const nodeIds = new Set([...entityIds, ...gridIds]); + + for (const dup of duplicates(data.sources.map((s) => s.source_id))) problems.push(`duplicate source_id ${dup}`); + for (const dup of duplicates(data.grids.map((g) => g.grid_id))) problems.push(`duplicate grid_id ${dup}`); + for (const dup of duplicates(data.entities.map((e) => e.entity_id))) problems.push(`duplicate entity_id ${dup}`); + for (const dup of duplicates(data.gaps.map((g) => g.gap_id))) problems.push(`duplicate gap_id ${dup}`); + for (const dup of duplicates(data.relationships.map((r) => r.rel_id))) problems.push(`duplicate rel_id ${dup}`); + + const requireSource = (id: string, where: string) => { + if (!sourceIds.has(id)) problems.push(`${where}: unknown source ${id}`); + }; + const requireNode = (id: string, where: string) => { + if (!nodeIds.has(id)) problems.push(`${where}: unknown entity/grid ${id}`); + }; + + for (const grid of data.grids) { + for (const id of grid.sources) requireSource(id, grid.grid_id); + for (const id of grid.live_data.source_ids ?? []) requireSource(id, `${grid.grid_id}.live_data`); + if (grid.parent_grid && !gridIds.has(grid.parent_grid)) { + problems.push(`${grid.grid_id}: unknown parent_grid ${grid.parent_grid}`); + } + } + for (const entity of data.entities) { + for (const id of entity.sources) requireSource(id, entity.entity_id); + if (entity.grid_id && !nodeIds.has(entity.grid_id)) { + problems.push(`${entity.entity_id}: unknown grid_id ${entity.grid_id}`); + } + if (entity.same_as && !nodeIds.has(entity.same_as)) { + problems.push(`${entity.entity_id}: unknown same_as ${entity.same_as}`); + } + } + for (const gap of data.gaps) { + requireNode(gap.entity_id, gap.gap_id); + if (gap.current_source) requireSource(gap.current_source, gap.gap_id); + if (gap.best_available_proxy) requireSource(gap.best_available_proxy.source_id, `${gap.gap_id}.proxy`); + } + for (const rel of data.relationships) { + requireNode(rel.from_id, rel.rel_id); + requireNode(rel.to_id, rel.rel_id); + requireSource(rel.source, rel.rel_id); + } + return problems; +} diff --git a/intelligence/packages/registry/src/resolve.ts b/intelligence/packages/registry/src/resolve.ts new file mode 100644 index 0000000..4fa8533 --- /dev/null +++ b/intelligence/packages/registry/src/resolve.ts @@ -0,0 +1,99 @@ +// Entity resolution: find records that likely describe the same real-world thing. +// Pass 1 — identical typed external IDs. Pass 2 — name similarity + geometry proximity. +// Candidates are REPORTED, never auto-merged; a human closes them via `same_as`. +import type { EntityRecord } from "@pact-tailor/ontology"; + +export interface ResolutionCandidate { + a: string; + b: string; + reason: string; + score: number; +} + +const MATCH_KEYS = ["imo", "mmsi", "aemo_station_id", "entsoe_eic", "eia_plant_code", "gem_id", "unlocode", "abn", "lei", "wikidata_qid"] as const; + +function normalise(name: string): string { + return name + .toLowerCase() + .replace(/\b(power station|power plant|ps|pty ltd|limited|ltd|corporation|corp)\b/g, "") + .replace(/[^a-z0-9]+/g, " ") + .trim(); +} + +function trigrams(s: string): Set { + const padded = ` ${s} `; + const grams = new Set(); + for (let i = 0; i < padded.length - 2; i++) grams.add(padded.slice(i, i + 3)); + return grams; +} + +function similarity(a: string, b: string): number { + const ta = trigrams(normalise(a)); + const tb = trigrams(normalise(b)); + if (ta.size === 0 || tb.size === 0) return 0; + let shared = 0; + for (const gram of ta) if (tb.has(gram)) shared++; + return (2 * shared) / (ta.size + tb.size); +} + +function pointOf(entity: EntityRecord): [number, number] | null { + const geom = entity.geometry; + if (geom && geom.type === "Point") return geom.coordinates as [number, number]; + return null; +} + +function kmBetween([lon1, lat1]: [number, number], [lon2, lat2]: [number, number]): number { + const rad = Math.PI / 180; + const dLat = (lat2 - lat1) * rad; + const dLon = (lon2 - lon1) * rad; + const a = + Math.sin(dLat / 2) ** 2 + Math.cos(lat1 * rad) * Math.cos(lat2 * rad) * Math.sin(dLon / 2) ** 2; + return 6371 * 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)); +} + +/** Pairwise scan. Fine at registry scale (thousands); production would block on typed keys first. */ +export function findResolutionCandidates( + entities: EntityRecord[], + options: { nameThreshold?: number; maxKm?: number } = {}, +): ResolutionCandidate[] { + const nameThreshold = options.nameThreshold ?? 0.82; + const maxKm = options.maxKm ?? 5; + const candidates: ResolutionCandidate[] = []; + + for (let i = 0; i < entities.length; i++) { + for (let j = i + 1; j < entities.length; j++) { + const a = entities[i]!; + const b = entities[j]!; + if (a.same_as === b.entity_id || b.same_as === a.entity_id) continue; + + let matched = false; + for (const key of MATCH_KEYS) { + const va = a.external_ids?.[key]; + const vb = b.external_ids?.[key]; + if (va !== undefined && vb !== undefined && va === vb) { + candidates.push({ a: a.entity_id, b: b.entity_id, reason: `shared external_ids.${key}=${String(va)}`, score: 1 }); + matched = true; + break; + } + } + if (matched || a.entity_type !== b.entity_type) continue; + + const nameScore = similarity(a.name, b.name); + if (nameScore < nameThreshold) continue; + const pa = pointOf(a); + const pb = pointOf(b); + if (pa && pb) { + const km = kmBetween(pa, pb); + if (km <= maxKm) { + candidates.push({ + a: a.entity_id, + b: b.entity_id, + reason: `name similarity ${nameScore.toFixed(2)} within ${km.toFixed(1)} km`, + score: nameScore, + }); + } + } + } + } + return candidates; +} diff --git a/intelligence/packages/registry/test/registry.test.mjs b/intelligence/packages/registry/test/registry.test.mjs new file mode 100644 index 0000000..0182848 --- /dev/null +++ b/intelligence/packages/registry/test/registry.test.mjs @@ -0,0 +1,159 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { loadRegistry, checkIntegrity, gridEntitiesFrom, findResolutionCandidates } from "../dist/index.js"; + +function writeFixtureTree(root) { + mkdirSync(join(root, "registries", "grids"), { recursive: true }); + mkdirSync(join(root, "registries", "sources"), { recursive: true }); + mkdirSync(join(root, "registries", "gaps"), { recursive: true }); + mkdirSync(join(root, "entities", "test"), { recursive: true }); + mkdirSync(join(root, "relationships", "test"), { recursive: true }); + + writeFileSync( + join(root, "registries", "sources", "test.yaml"), + `records: + - source_id: source:test:feed + publisher: Test Publisher + dataset: Test Dataset + api_type: rest + url: https://example.com/api + coverage: { grids: [grid:test], countries: [au] } + data_class: live + licensing: { class: open, notes: test } + verification_status: documented_only + last_checked: "2026-08-12" +`, + ); + writeFileSync( + join(root, "registries", "grids", "test.yaml"), + `records: + - grid_id: grid:test + name: Test Grid + kind: isolated_grid + countries: [au] + frequency_hz: 50 + live_data: { availability: rich, notes: test, source_ids: [source:test:feed] } + sources: [source:test:feed] +`, + ); + writeFileSync( + join(root, "entities", "test", "entities.yaml"), + `records: + - entity_id: gen:test:alpha + entity_type: Generator + name: Alpha Power Station + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [151.0, -23.0] } + country: au + grid_id: grid:test + properties: { fuel: coal, capacity_mw: 100 } + external_ids: { aemo_station_id: ALPHA } + sources: [source:test:feed] + - entity_id: gen:test:alpha-dup + entity_type: Generator + name: Alpha Power Stn + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [151.001, -23.001] } + country: au + grid_id: grid:test + external_ids: { aemo_station_id: ALPHA } + sources: [source:test:feed] +`, + ); + writeFileSync( + join(root, "registries", "gaps", "test.yaml"), + `records: + - gap_id: gap:test:alpha-metric + entity_id: gen:test:alpha + desired_metric: power.output.mw + commercial_value: high + strategic_value: high + priority: P1 + status: open +`, + ); + writeFileSync( + join(root, "relationships", "test", "rels.yaml"), + `records: + - rel_id: rel:alpha-generates-test + rel_type: GENERATES + from_id: gen:test:alpha + to_id: grid:test + confidence: 1.0 + source: source:test:feed + method: declared +`, + ); +} + +test("loads a valid fixture tree with no errors and full integrity", () => { + const root = mkdtempSync(join(tmpdir(), "pt-registry-")); + writeFixtureTree(root); + const { data, errors } = loadRegistry(root); + assert.deepEqual(errors, []); + assert.equal(data.grids.length, 1); + assert.equal(data.sources.length, 1); + assert.equal(data.gaps.length, 1); + assert.equal(data.entities.length, 2); + assert.equal(data.relationships.length, 1); + assert.deepEqual(checkIntegrity(data), []); +}); + +test("schema-invalid records are reported with file context and excluded", () => { + const root = mkdtempSync(join(tmpdir(), "pt-registry-")); + writeFixtureTree(root); + writeFileSync( + join(root, "entities", "test", "broken.yaml"), + `records: + - entity_id: not a valid id + entity_type: Generator + name: Broken + data_class: structural + observability: KNOWN_LIVE + sources: [source:test:feed] +`, + ); + const { data, errors } = loadRegistry(root); + assert.equal(data.entities.length, 2); + assert.equal(errors.length, 1); + assert.match(errors[0], /broken\.yaml/); +}); + +test("integrity check flags dangling references", () => { + const root = mkdtempSync(join(tmpdir(), "pt-registry-")); + writeFixtureTree(root); + writeFileSync( + join(root, "relationships", "test", "dangling.yaml"), + `records: + - rel_id: rel:dangling + rel_type: SUPPLIES + from_id: gen:test:alpha + to_id: smelter:test:missing + confidence: 0.9 + source: source:test:feed + method: manual +`, + ); + const { data } = loadRegistry(root); + const problems = checkIntegrity(data); + assert.ok(problems.some((p) => p.includes("smelter:test:missing"))); +}); + +test("grid entities materialise and shared-external-id duplicates surface", () => { + const root = mkdtempSync(join(tmpdir(), "pt-registry-")); + writeFixtureTree(root); + const { data } = loadRegistry(root); + const gridEntities = gridEntitiesFrom(data.grids); + assert.equal(gridEntities.length, 1); + assert.equal(gridEntities[0].entity_id, "grid:test"); + assert.equal(gridEntities[0].entity_type, "Grid"); + assert.equal(gridEntities[0].observability, "KNOWN_LIVE"); + + const candidates = findResolutionCandidates(data.entities); + assert.ok(candidates.some((c) => c.reason.includes("aemo_station_id"))); +}); From 19413801f035f6a34aff5b71b64f4e5b8b6fafef Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 01:46:00 +0000 Subject: [PATCH 06/16] feat(intelligence): real-time state engine (phase 14) Observations in, per-(entity, metric) current state out: event-time latest-wins, late data lands in history without touching state, corrections supersede via corrects/source_sequence and emit state.corrected with the previous value, tumbling 5-min/1-h windows with watermark-based revision, typed delta bus, gold snapshots, and boot-time silver replay. Canonical entities are never mutated. 18 tests. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014yVGqBtvM1zNAd9tvUVNzm --- intelligence/packages/state/package.json | 3 +- intelligence/packages/state/src/bus.ts | 31 +++ intelligence/packages/state/src/current.ts | 145 ++++++++++ intelligence/packages/state/src/index.ts | 152 ++++++++++- .../packages/state/src/observations.ts | Bin 0 -> 3349 bytes intelligence/packages/state/src/windows.ts | 151 +++++++++++ .../packages/state/test/state.test.mjs | 254 ++++++++++++++++++ .../packages/state/test/windows.test.mjs | 127 +++++++++ 8 files changed, 860 insertions(+), 3 deletions(-) create mode 100644 intelligence/packages/state/src/bus.ts create mode 100644 intelligence/packages/state/src/current.ts create mode 100644 intelligence/packages/state/src/observations.ts create mode 100644 intelligence/packages/state/src/windows.ts create mode 100644 intelligence/packages/state/test/state.test.mjs create mode 100644 intelligence/packages/state/test/windows.test.mjs diff --git a/intelligence/packages/state/package.json b/intelligence/packages/state/package.json index 1cdef75..5f1e0ef 100644 --- a/intelligence/packages/state/package.json +++ b/intelligence/packages/state/package.json @@ -13,7 +13,8 @@ "node": ">=20" }, "scripts": { - "build": "tsc -p tsconfig.json" + "build": "tsc -p tsconfig.json", + "test": "npm run build && node --test \"test/*.test.mjs\"" }, "dependencies": { "@pact-tailor/ontology": "*", diff --git a/intelligence/packages/state/src/bus.ts b/intelligence/packages/state/src/bus.ts new file mode 100644 index 0000000..0a6ac7a --- /dev/null +++ b/intelligence/packages/state/src/bus.ts @@ -0,0 +1,31 @@ +// StateBus — the typed delta channel between the state engine and its +// consumers (API SSE, intel rules, the globe). One event shape, one event +// name; subscribers get every state.updated / state.corrected in ingest order. +import { EventEmitter } from "node:events"; +import type { CurrentMetricState } from "./current.js"; + +export type StateDeltaType = "state.updated" | "state.corrected"; + +export interface StateDelta { + type: StateDeltaType; + /** The new held state for (entity_id, metric). */ + state: CurrentMetricState; + /** The state it replaced — null on first sight of the key. */ + previous: CurrentMetricState | null; +} + +const DELTA_EVENT = "delta"; + +export class StateBus extends EventEmitter { + emitDelta(d: StateDelta): boolean { + return this.emit(DELTA_EVENT, d); + } + + /** Subscribe to deltas. Returns the unsubscribe function. */ + onDelta(cb: (d: StateDelta) => void): () => void { + this.on(DELTA_EVENT, cb); + return () => { + this.off(DELTA_EVENT, cb); + }; + } +} diff --git a/intelligence/packages/state/src/current.ts b/intelligence/packages/state/src/current.ts new file mode 100644 index 0000000..ca1af9d --- /dev/null +++ b/intelligence/packages/state/src/current.ts @@ -0,0 +1,145 @@ +// CurrentStateProjection — the per-(entity_id, metric) "what is it doing now" +// view. Canonical entities are never mutated: this projection is the only +// holder of live state, and consumers compose it with structural records at +// read time. Semantics (phase 14 of the handoff): +// +// - Event-time latest wins: strictly newer event_time replaces the held +// value and yields a "state.updated" delta. +// - Late data (event_time equal-or-older, not a correction): no change, +// no delta — callers keep it in history only. +// - Corrections (is_correction: true AND corrects === held observation_id +// OR source_sequence === held source_sequence): supersede the held value +// EVEN at equal-or-older event_time, yielding "state.corrected" with the +// previous value attached. +// - Duplicates (same observation_id as held): ignored. +// +// Staleness is derived, never stored: staleness(now) computes it on demand. +import type { ObservationRecord, ObservationValue, Quality } from "@pact-tailor/ontology"; +import type { StateDelta } from "./bus.js"; + +export interface CurrentMetricState { + entity_id: string; + metric: string; + value: ObservationValue; + unit?: string; + event_time: string; + ingest_time: string; + source_id: string; + feed_id: string; + quality: Quality; + observation_id: string; +} + +interface Held { + state: CurrentMetricState; + /** Publisher run/interval id of the held observation — corrections match on it. */ + source_sequence?: string; +} + +function keyOf(entityId: string, metric: string): string { + return `${entityId} ${metric}`; +} + +function eventMs(iso: string): number { + return Date.parse(iso); +} + +function toState(obs: ObservationRecord): CurrentMetricState { + return { + entity_id: obs.entity_id, + metric: obs.metric, + value: obs.value, + ...(obs.unit !== undefined ? { unit: obs.unit } : {}), + event_time: obs.event_time, + ingest_time: obs.ingest_time, + source_id: obs.source_id, + feed_id: obs.feed_id, + quality: obs.quality, + observation_id: obs.observation_id, + }; +} + +export class CurrentStateProjection { + private readonly held = new Map(); + + /** + * Fold one observation into the projection. Returns the resulting delta, + * or null when the observation changes nothing (duplicate or late). + */ + apply(obs: ObservationRecord): StateDelta | null { + const key = keyOf(obs.entity_id, obs.metric); + const prev = this.held.get(key); + + if (!prev) { + const state = toState(obs); + this.setHeld(key, state, obs); + return { type: "state.updated", state, previous: null }; + } + + // Duplicate of the held observation: ignore. + if (obs.observation_id === prev.state.observation_id) return null; + + // Correction of the held observation supersedes even at equal-or-older + // event_time. It must explicitly reference what it corrects — by the held + // observation_id, or by sharing the held publisher source_sequence. + const correctsHeld = + obs.is_correction === true && + ((obs.corrects != null && obs.corrects === prev.state.observation_id) || + (obs.source_sequence !== undefined && obs.source_sequence === prev.source_sequence)); + if (correctsHeld) { + const state = toState(obs); + this.setHeld(key, state, obs); + return { type: "state.corrected", state, previous: prev.state }; + } + + // Event-time latest wins. + if (eventMs(obs.event_time) > eventMs(prev.state.event_time)) { + const state = toState(obs); + this.setHeld(key, state, obs); + return { type: "state.updated", state, previous: prev.state }; + } + + // Late (or equal-time non-correction): history only, no state change. + return null; + } + + get(entityId: string, metric: string): CurrentMetricState | null { + return this.held.get(keyOf(entityId, metric))?.state ?? null; + } + + /** All held metric states for one entity, sorted by metric. */ + forEntity(entityId: string): CurrentMetricState[] { + const out: CurrentMetricState[] = []; + for (const { state } of this.held.values()) { + if (state.entity_id === entityId) out.push(state); + } + return out.sort((a, b) => a.metric.localeCompare(b.metric)); + } + + /** Every held state, sorted by (entity_id, metric) for determinism. */ + all(): CurrentMetricState[] { + const out: CurrentMetricState[] = []; + for (const { state } of this.held.values()) out.push(state); + return out.sort( + (a, b) => a.entity_id.localeCompare(b.entity_id) || a.metric.localeCompare(b.metric), + ); + } + + /** + * Milliseconds between `now` and the held event_time — derived on demand, + * never stored. Null when no state is held for the key. + */ + staleness(entityId: string, metric: string, now: string | number | Date = Date.now()): number | null { + const state = this.get(entityId, metric); + if (!state) return null; + const nowMs = typeof now === "number" ? now : typeof now === "string" ? Date.parse(now) : now.getTime(); + return nowMs - eventMs(state.event_time); + } + + private setHeld(key: string, state: CurrentMetricState, obs: ObservationRecord): void { + this.held.set( + key, + obs.source_sequence !== undefined ? { state, source_sequence: obs.source_sequence } : { state }, + ); + } +} diff --git a/intelligence/packages/state/src/index.ts b/intelligence/packages/state/src/index.ts index 0dbc94b..4bcc4b0 100644 --- a/intelligence/packages/state/src/index.ts +++ b/intelligence/packages/state/src/index.ts @@ -1,2 +1,150 @@ -// @pact-tailor/state — real-time state engine. Populated in the state commit. -export {}; +// @pact-tailor/state — the real-time state engine (handoff phase 14). +// Observations in, per-(entity_id, metric) current state + bounded history + +// tumbling-window aggregates + typed deltas out. Canonical entities are never +// mutated; the API composes this state with structural records at read time. +import type { ObservationRecord } from "@pact-tailor/ontology"; +import type { MedallionStore } from "@pact-tailor/store"; +import { ObservationLog, DEFAULT_RETENTION_PER_KEY, type HistoryRange } from "./observations.js"; +import { CurrentStateProjection, type CurrentMetricState } from "./current.js"; +import { + WindowAggregator, + DEFAULT_ALLOWED_LATENESS_MS, + type WindowAggregate, + type WindowSizeMs, +} from "./windows.js"; +import { StateBus, type StateDelta } from "./bus.js"; + +export { ObservationLog, DEFAULT_RETENTION_PER_KEY, type HistoryRange } from "./observations.js"; +export { CurrentStateProjection, type CurrentMetricState } from "./current.js"; +export { + WindowAggregator, + WINDOW_5M, + WINDOW_1H, + WINDOW_SIZES_MS, + DEFAULT_ALLOWED_LATENESS_MS, + type WindowSizeMs, + type WindowAggregate, + type WindowAddResult, +} from "./windows.js"; +export { StateBus, type StateDelta, type StateDeltaType } from "./bus.js"; + +export interface StateEngineOptions { + /** Ring-buffer depth per (entity_id, metric); default 576 (~48 h of 5-min data). */ + retentionPerKey?: number; + /** Watermark lag for window aggregation; default 600 000 ms (10 minutes). */ + allowedLatenessMs?: number; +} + +export interface IngestResult { + /** Deltas emitted, in ingest order (each was also emitted on the bus). */ + deltas: StateDelta[]; + /** Observations recorded to history only (event_time not newer than held, no correction match). */ + late: number; + /** Observations dropped because their observation_id was already seen for the key. */ + duplicates: number; + /** Subset of deltas that were state.corrected. */ + corrections: number; +} + +export type StateSnapshot = { at: string; states: CurrentMetricState[] }; + +export class StateEngine { + readonly log: ObservationLog; + readonly projection: CurrentStateProjection; + readonly aggregator: WindowAggregator; + readonly bus: StateBus; + + constructor(opts: StateEngineOptions = {}) { + this.log = new ObservationLog(opts.retentionPerKey ?? DEFAULT_RETENTION_PER_KEY); + this.projection = new CurrentStateProjection(); + this.aggregator = new WindowAggregator( + opts.allowedLatenessMs !== undefined ? { allowedLatenessMs: opts.allowedLatenessMs } : {}, + ); + this.bus = new StateBus(); + } + + /** + * Fold a batch of observations into the engine, in the given order. + * Duplicates are dropped before touching history, windows, or state. + * Every returned delta was also emitted on the bus, in the same order. + */ + ingest(observations: ObservationRecord[]): IngestResult { + const deltas: StateDelta[] = []; + let late = 0; + let duplicates = 0; + let corrections = 0; + + for (const obs of observations) { + const heldId = this.projection.get(obs.entity_id, obs.metric)?.observation_id; + if (obs.observation_id === heldId || this.log.has(obs.entity_id, obs.metric, obs.observation_id)) { + duplicates += 1; + continue; + } + + this.log.record(obs); + // Every non-duplicate numeric observation lands in its event-time + // window (late arrivals patch closed windows as revisions). + this.aggregator.add(obs); + + const delta = this.projection.apply(obs); + if (delta === null) { + late += 1; + continue; + } + if (delta.type === "state.corrected") corrections += 1; + deltas.push(delta); + this.bus.emitDelta(delta); + } + + return { deltas, late, duplicates, corrections }; + } + + /** Held state for one metric ([] when unknown), or all metrics of an entity sorted by metric. */ + current(entityId: string, metric?: string): CurrentMetricState[] { + if (metric !== undefined) { + const state = this.projection.get(entityId, metric); + return state ? [state] : []; + } + return this.projection.forEntity(entityId); + } + + /** Retained observations for the key, sorted by event_time; range bounds are inclusive. */ + history(entityId: string, metric: string, range?: { from?: string; to?: string }): ObservationRecord[] { + return this.log.history(entityId, metric, range as HistoryRange | undefined); + } + + /** Tumbling-window aggregates (5 min or 1 h) for the key, sorted by window_start. */ + windows(entityId: string, metric: string, windowMs: WindowSizeMs): WindowAggregate[] { + return this.aggregator.windows(entityId, metric, windowMs); + } + + /** Derived, never stored: ms between `now` and the held event_time; null when the key is unknown. */ + staleness(entityId: string, metric: string, now: string | number | Date = Date.now()): number | null { + return this.projection.staleness(entityId, metric, now); + } + + /** Subscribe to every delta the engine emits. Returns the unsubscribe function. */ + onDelta(cb: (d: StateDelta) => void): () => void { + return this.bus.onDelta(cb); + } + + /** Deterministic full-state snapshot, shaped for GoldStore.snapshotState(snap, snap.at). */ + snapshot(): StateSnapshot { + return { at: new Date().toISOString(), states: this.projection.all() }; + } + + /** + * Boot-time replay from silver: reads the given feeds (default: all), + * orders observations by ingest_time (stable, so same-instant arrivals keep + * file order), and ingests them — corrections and late data resolve exactly + * as they did live. Returns the number of observations replayed. + */ + loadFromSilver(store: MedallionStore, feedIds?: string[]): number { + const feeds = feedIds ?? store.silver.listFeeds(); + const observations: ObservationRecord[] = []; + for (const feedId of feeds) observations.push(...store.silver.readFeed(feedId)); + observations.sort((a, b) => Date.parse(a.ingest_time) - Date.parse(b.ingest_time)); + this.ingest(observations); + return observations.length; + } +} diff --git a/intelligence/packages/state/src/observations.ts b/intelligence/packages/state/src/observations.ts new file mode 100644 index 0000000000000000000000000000000000000000..f066b870bd141e48099ce8c9b03d557de9c6ee66 GIT binary patch literal 3349 zcma)9?QR=45bSR~#U=>aJ3~GtK$8Y`Y}Y_s)X0Ye$Nf}>9nS1-CKEcFI%%(kS4O`!Gy3=Me`soIUC4rz-d)Iru`8M@ z=lSPST1uq$%C9a}F{Xv|R^@3nnoQ`JtkN@@)@6xZN|UfwU5kp0E%=z0H_~|$%cW%W zF`2611$nJ6Jl%JDeC|tAqhg^{=$VpGmP*E={l?yM)^zkADhg&2S(#zb7X(jpI zv*ZcD=Bkp6<>vshav|^y{#tBB{6=SVE=6BHr~mqWH{wW z8y~)7N)S&8%p5re>YS>sIeVAUOIeD#^8JnU^rvTmL8Qh|$zMM{r#S;W-CZbMBe_C& z0XEfQiHOLrmXdCVFLWMnT2pDwf*$_76uIB^LRH32jP|B7v(>}>(MWy@0J+g1_~pqj z$G^S4xI90(IC*<-Sc<_1XynUK)~(kMB7e@4%rJ${mAv@^j$4)kb`9CV^*`&&pg)bNc+-VAWoqfsC2 zE4gCnws9VB=#B-gAh=KTm>-WHwGkeq08!xG2a;-{jw0UMrvAgnHiyBXpu@bj7TT$x zT0M|X0|=$=(R=A%L?RCNN64V8bG2;!`4e05s1Y_;}ke^1*FAyY(OTTNGwZlfPh zi}iPpZlfXBSDW}j)o)y)oM|xtrt8HNOuG}4C#9h3#T(;KmD5*|~0q{9TLL|cbCTee|Al|(_mw$5R+q}%aMT41&NaMKs{ zorc_qR7uH$w;|h1xl^coCbs}n`uw@~YP^JA5K0LX#DDzUnwyYU@WqL>29quV=~t*-(_MC)*`NA1nJEl9Wf9AjiUp`~ci3tP)r z`bg-Zu48d!EH0|}7bsEurLjS-&|st0xp4ohMO5#;4)>CUs;0(N5|>BTa1E<8B*DVS zX3{5{a&KjxrbiiUlJWH$;3*v)Q7I}Xhq*F2J9Hen$=zhVA>|$0>IY=nccpP`HoVZ+ zfe6$!D~WJLpU>3_lXw&>_)2f+5q6%FV&w>40FVl!Gcm`W~ z`N#dYO-tOP4H1)>e_jsDH=J?85+bj=rFZTRHbCS7UDEU|pL z$4BHi6038ztlgX`YG7M1Q53v-7c7Pr#)5cpgHlO7!`yKgWA0wQ&pWc)OetV&5P_Sk z5_z{-b4uMb(}q(>kS*Kw`=g3ZZ<6)!Fe7%RK5wNmpa&xCeLKd4v^~I3;^Ihz2IG}< zZAZq_@GSQY7Ps8~_cn)t1G-&9fM>n?<4p45W8?Tp7qY)kT!Svx+dk6Q3$EcIw%Vo% zggN33_mvP-7UrJ^2M5HHY8krtQJh22=K3uuai6HFcts4&n*k0g#n(~1*~Dmq>qQX! zM3j}P!t|5`LDIot*EQM%JYsMAFw3)_G&}voPQ|GqLJo^Vr|-Wfk46-i@3RCOn%N79 zZ61TFk?ofb-4Pt)tV0FEhm0kaBw|d{l+$TL7sJ45w_BeM*F#2~eq-m2gYx#owV~vu m13C34925SV(f5{CF!xU|-3-&0p0Br>P5Yz6JdUPYkNyLFBULc~ literal 0 HcmV?d00001 diff --git a/intelligence/packages/state/src/windows.ts b/intelligence/packages/state/src/windows.ts new file mode 100644 index 0000000..4e0ff53 --- /dev/null +++ b/intelligence/packages/state/src/windows.ts @@ -0,0 +1,151 @@ +// WindowAggregator — tumbling 5-minute and 1-hour windows per +// (entity_id, metric) over numeric observation values, computing +// {min, max, mean, last, count}. Windows are aligned to the epoch +// (start = floor(event_time / size) * size). +// +// The watermark is the maximum event_time seen across all observations minus +// allowedLatenessMs (default 10 minutes). A window whose end is at or before +// the watermark is closed; an observation that lands in a closed window still +// patches it — the add() result flags it as a revision so downstream can +// re-emit the corrected aggregate. Non-numeric values are skipped. +import type { ObservationRecord } from "@pact-tailor/ontology"; + +export const WINDOW_5M = 300_000; +export const WINDOW_1H = 3_600_000; +export type WindowSizeMs = typeof WINDOW_5M | typeof WINDOW_1H; +export const WINDOW_SIZES_MS: readonly WindowSizeMs[] = [WINDOW_5M, WINDOW_1H]; + +export const DEFAULT_ALLOWED_LATENESS_MS = 600_000; + +export interface WindowAggregate { + entity_id: string; + metric: string; + window_ms: WindowSizeMs; + /** Inclusive window start, ISO 8601. */ + window_start: string; + /** Exclusive window end, ISO 8601. */ + window_end: string; + min: number; + max: number; + mean: number; + /** Value of the latest-event_time observation in the window (arrival breaks ties). */ + last: number; + count: number; +} + +export interface WindowAddResult { + /** False when the value was non-numeric (or event_time unparseable) and skipped. */ + accepted: boolean; + /** True when the observation landed in at least one already-closed window. */ + revised: boolean; + /** Fresh snapshots of the windows this observation landed in (one per size). */ + aggregates: WindowAggregate[]; +} + +interface WindowState { + min: number; + max: number; + sum: number; + count: number; + last: number; + lastEventMs: number; +} + +function keyOf(entityId: string, metric: string): string { + return `${entityId} ${metric}`; +} + +export class WindowAggregator { + /** size → key → windowStartMs → running aggregate. */ + private readonly windowsBySize = new Map>>(); + private readonly allowedLatenessMs: number; + private maxEventMs: number | null = null; + + constructor(opts: { allowedLatenessMs?: number } = {}) { + this.allowedLatenessMs = opts.allowedLatenessMs ?? DEFAULT_ALLOWED_LATENESS_MS; + for (const size of WINDOW_SIZES_MS) this.windowsBySize.set(size, new Map()); + } + + /** Current watermark in epoch ms (max seen event_time − allowed lateness), or null before any data. */ + watermarkMs(): number | null { + return this.maxEventMs === null ? null : this.maxEventMs - this.allowedLatenessMs; + } + + /** + * Fold one observation into its 5-minute and 1-hour windows. Closed-window + * membership is judged against the watermark as it stood BEFORE this + * observation, so an observation never closes its own window. + */ + add(obs: ObservationRecord): WindowAddResult { + const value = obs.value; + const eventMs = Date.parse(obs.event_time); + if (typeof value !== "number" || !Number.isFinite(value) || Number.isNaN(eventMs)) { + return { accepted: false, revised: false, aggregates: [] }; + } + + const priorWatermark = this.watermarkMs(); + const key = keyOf(obs.entity_id, obs.metric); + const aggregates: WindowAggregate[] = []; + let revised = false; + + for (const size of WINDOW_SIZES_MS) { + const startMs = Math.floor(eventMs / size) * size; + const endMs = startMs + size; + if (priorWatermark !== null && endMs <= priorWatermark) revised = true; + + const byKey = this.windowsBySize.get(size) as Map>; + let byStart = byKey.get(key); + if (!byStart) { + byStart = new Map(); + byKey.set(key, byStart); + } + const win = byStart.get(startMs); + if (!win) { + byStart.set(startMs, { min: value, max: value, sum: value, count: 1, last: value, lastEventMs: eventMs }); + } else { + win.min = Math.min(win.min, value); + win.max = Math.max(win.max, value); + win.sum += value; + win.count += 1; + if (eventMs >= win.lastEventMs) { + win.last = value; + win.lastEventMs = eventMs; + } + } + aggregates.push(this.snapshot(obs.entity_id, obs.metric, size, startMs, byStart.get(startMs) as WindowState)); + } + + if (this.maxEventMs === null || eventMs > this.maxEventMs) this.maxEventMs = eventMs; + return { accepted: true, revised, aggregates }; + } + + /** All aggregates for the key at one window size, sorted by window_start. */ + windows(entityId: string, metric: string, windowMs: WindowSizeMs): WindowAggregate[] { + const byStart = this.windowsBySize.get(windowMs)?.get(keyOf(entityId, metric)); + if (!byStart) return []; + return [...byStart.entries()] + .sort(([a], [b]) => a - b) + .map(([startMs, win]) => this.snapshot(entityId, metric, windowMs, startMs, win)); + } + + private snapshot( + entityId: string, + metric: string, + size: WindowSizeMs, + startMs: number, + win: WindowState, + ): WindowAggregate { + return { + entity_id: entityId, + metric, + window_ms: size, + window_start: new Date(startMs).toISOString(), + window_end: new Date(startMs + size).toISOString(), + min: win.min, + max: win.max, + mean: win.sum / win.count, + last: win.last, + count: win.count, + }; + } +} diff --git a/intelligence/packages/state/test/state.test.mjs b/intelligence/packages/state/test/state.test.mjs new file mode 100644 index 0000000..2ddbf44 --- /dev/null +++ b/intelligence/packages/state/test/state.test.mjs @@ -0,0 +1,254 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { MedallionStore } from "@pact-tailor/store"; +import { StateEngine } from "../dist/index.js"; + +let seq = 0; +const obs = (entity, metric, eventTime, value, extra = {}) => ({ + observation_id: `obs-${String(++seq).padStart(4, "0")}`, + entity_id: entity, + metric, + value, + unit: "MW", + event_time: eventTime, + ingest_time: extra.ingest_time ?? `2026-08-12T02:${String(seq % 60).padStart(2, "0")}:00Z`, + source_id: "source:test:feed", + feed_id: "feed:test:a", + quality: "good", + ...extra, +}); + +test("event-time latest wins: newer observation replaces held state, emits state.updated", () => { + const engine = new StateEngine(); + const a = obs("gen:test:alpha", "power.output.mw", "2026-08-12T01:00:00Z", 100); + const b = obs("gen:test:alpha", "power.output.mw", "2026-08-12T01:05:00Z", 110); + + const r1 = engine.ingest([a]); + assert.equal(r1.deltas.length, 1); + assert.equal(r1.deltas[0].type, "state.updated"); + assert.equal(r1.deltas[0].previous, null); + + const r2 = engine.ingest([b]); + assert.equal(r2.deltas.length, 1); + assert.equal(r2.deltas[0].type, "state.updated"); + assert.equal(r2.deltas[0].state.value, 110); + assert.equal(r2.deltas[0].previous.value, 100); + assert.equal(r2.deltas[0].previous.observation_id, a.observation_id); + + const current = engine.current("gen:test:alpha", "power.output.mw"); + assert.equal(current.length, 1); + assert.equal(current[0].value, 110); + assert.equal(current[0].observation_id, b.observation_id); +}); + +test("late data: older event_time goes to history only — no state change, no delta", () => { + const engine = new StateEngine(); + engine.ingest([obs("gen:test:alpha", "power.output.mw", "2026-08-12T01:05:00Z", 110)]); + + const lateObs = obs("gen:test:alpha", "power.output.mw", "2026-08-12T01:00:00Z", 90); + let busCalls = 0; + engine.onDelta(() => busCalls++); + const r = engine.ingest([lateObs]); + + assert.equal(r.deltas.length, 0); + assert.equal(r.late, 1); + assert.equal(busCalls, 0); + assert.equal(engine.current("gen:test:alpha", "power.output.mw")[0].value, 110); + + // ...but it IS in history, which is sorted by event_time. + const history = engine.history("gen:test:alpha", "power.output.mw"); + assert.deepEqual(history.map((o) => o.value), [90, 110]); +}); + +test("correction supersedes at equal event_time via corrects", () => { + const engine = new StateEngine(); + const orig = obs("gen:test:alpha", "power.output.mw", "2026-08-12T01:05:00Z", 110); + engine.ingest([orig]); + + const corr = obs("gen:test:alpha", "power.output.mw", "2026-08-12T01:05:00Z", 105, { + is_correction: true, + corrects: orig.observation_id, + }); + const r = engine.ingest([corr]); + + assert.equal(r.corrections, 1); + assert.equal(r.late, 0); + assert.equal(r.deltas.length, 1); + assert.equal(r.deltas[0].type, "state.corrected"); + assert.equal(r.deltas[0].state.value, 105); + assert.equal(r.deltas[0].previous.value, 110); + assert.equal(engine.current("gen:test:alpha", "power.output.mw")[0].value, 105); +}); + +test("correction supersedes at older event_time via source_sequence", () => { + const engine = new StateEngine(); + const orig = obs("gen:test:beta", "power.output.mw", "2026-08-12T01:10:00Z", 50, { + source_sequence: "DISPATCH-RUN-42", + }); + engine.ingest([orig]); + + const corr = obs("gen:test:beta", "power.output.mw", "2026-08-12T01:05:00Z", 55, { + is_correction: true, + source_sequence: "DISPATCH-RUN-42", + }); + const r = engine.ingest([corr]); + + assert.equal(r.corrections, 1); + assert.equal(r.deltas[0].type, "state.corrected"); + assert.equal(r.deltas[0].previous.observation_id, orig.observation_id); + assert.equal(engine.current("gen:test:beta", "power.output.mw")[0].value, 55); +}); + +test("a correction that matches neither corrects nor source_sequence follows time rules", () => { + const engine = new StateEngine(); + engine.ingest([ + obs("gen:test:beta", "power.output.mw", "2026-08-12T01:10:00Z", 50, { source_sequence: "RUN-1" }), + ]); + const stray = obs("gen:test:beta", "power.output.mw", "2026-08-12T01:00:00Z", 999, { + is_correction: true, + corrects: "obs-does-not-match", + source_sequence: "RUN-0", + }); + const r = engine.ingest([stray]); + assert.equal(r.deltas.length, 0); + assert.equal(r.late, 1); + assert.equal(engine.current("gen:test:beta", "power.output.mw")[0].value, 50); +}); + +test("duplicate observation_id is ignored everywhere", () => { + const engine = new StateEngine(); + const a = obs("gen:test:alpha", "power.output.mw", "2026-08-12T01:00:00Z", 100); + engine.ingest([a]); + const r = engine.ingest([a]); + + assert.equal(r.duplicates, 1); + assert.equal(r.deltas.length, 0); + assert.equal(r.late, 0); + assert.equal(engine.history("gen:test:alpha", "power.output.mw").length, 1); + assert.equal(engine.windows("gen:test:alpha", "power.output.mw", 300000)[0].count, 1); +}); + +test("deltas are emitted on the bus in ingest order and match the returned array", () => { + const engine = new StateEngine(); + const seen = []; + const unsubscribe = engine.onDelta((d) => seen.push(d)); + + const batch = [ + obs("gen:test:alpha", "power.output.mw", "2026-08-12T01:00:00Z", 1), + obs("gen:test:beta", "grid.demand.mw", "2026-08-12T01:00:00Z", 2), + obs("gen:test:alpha", "power.output.mw", "2026-08-12T01:05:00Z", 3), + ]; + const r = engine.ingest(batch); + + assert.equal(r.deltas.length, 3); + assert.deepEqual(seen, r.deltas); + assert.deepEqual( + seen.map((d) => d.state.observation_id), + batch.map((o) => o.observation_id), + ); + + unsubscribe(); + engine.ingest([obs("gen:test:alpha", "power.output.mw", "2026-08-12T01:10:00Z", 4)]); + assert.equal(seen.length, 3); +}); + +test("staleness is derived from event_time on demand, never stored", () => { + const engine = new StateEngine(); + engine.ingest([obs("gen:test:alpha", "power.output.mw", "2026-08-12T01:00:00Z", 100)]); + assert.equal(engine.staleness("gen:test:alpha", "power.output.mw", "2026-08-12T01:07:30Z"), 450000); + assert.equal(engine.staleness("gen:test:alpha", "no.such.metric"), null); + assert.equal("staleness" in engine.current("gen:test:alpha", "power.output.mw")[0], false); +}); + +test("retentionPerKey bounds history per (entity, metric), keeping newest arrivals", () => { + const engine = new StateEngine({ retentionPerKey: 3 }); + for (let i = 1; i <= 5; i++) { + engine.ingest([obs("gen:test:alpha", "power.output.mw", `2026-08-12T01:0${i}:00Z`, i * 10)]); + } + const history = engine.history("gen:test:alpha", "power.output.mw"); + assert.deepEqual(history.map((o) => o.value), [30, 40, 50]); + // Current state is untouched by eviction. + assert.equal(engine.current("gen:test:alpha", "power.output.mw")[0].value, 50); +}); + +test("history honours inclusive from/to bounds; current(entityId) lists all metrics", () => { + const engine = new StateEngine(); + engine.ingest([ + obs("gen:test:alpha", "power.output.mw", "2026-08-12T01:00:00Z", 1), + obs("gen:test:alpha", "power.output.mw", "2026-08-12T01:05:00Z", 2), + obs("gen:test:alpha", "power.output.mw", "2026-08-12T01:10:00Z", 3), + obs("gen:test:alpha", "power.capacity_factor", "2026-08-12T01:00:00Z", 0.4), + ]); + const ranged = engine.history("gen:test:alpha", "power.output.mw", { + from: "2026-08-12T01:05:00Z", + to: "2026-08-12T01:10:00Z", + }); + assert.deepEqual(ranged.map((o) => o.value), [2, 3]); + + const all = engine.current("gen:test:alpha"); + assert.deepEqual(all.map((s) => s.metric), ["power.capacity_factor", "power.output.mw"]); +}); + +test("snapshot round-trips through GoldStore.snapshotState", () => { + const store = new MedallionStore(mkdtempSync(join(tmpdir(), "pt-state-"))); + const engine = new StateEngine(); + engine.ingest([ + obs("gen:test:alpha", "power.output.mw", "2026-08-12T01:00:00Z", 100), + obs("gen:test:beta", "grid.demand.mw", "2026-08-12T01:00:00Z", 7000), + ]); + + const snap = engine.snapshot(); + assert.ok(/^\d{4}-\d{2}-\d{2}T/.test(snap.at)); + assert.equal(snap.states.length, 2); + // Deterministic order: (entity_id, metric). + assert.deepEqual(snap.states.map((s) => s.entity_id), ["gen:test:alpha", "gen:test:beta"]); + + const path = store.gold.snapshotState(snap, snap.at); + assert.deepEqual(JSON.parse(readFileSync(path, "utf8")), snap); +}); + +test("loadFromSilver replays a real MedallionStore feed with live semantics", () => { + const store = new MedallionStore(mkdtempSync(join(tmpdir(), "pt-state-"))); + const orig = obs("gen:test:alpha", "power.output.mw", "2026-08-11T23:55:00Z", 80, { + ingest_time: "2026-08-11T23:56:00Z", + source_sequence: "RUN-A", + }); + const newer = obs("gen:test:alpha", "power.output.mw", "2026-08-12T00:00:00Z", 90, { + ingest_time: "2026-08-12T00:01:00Z", + source_sequence: "RUN-B", + }); + const corr = obs("gen:test:alpha", "power.output.mw", "2026-08-12T00:00:00Z", 92, { + ingest_time: "2026-08-12T00:06:00Z", + is_correction: true, + source_sequence: "RUN-B", + }); + const lateArrival = obs("gen:test:alpha", "power.output.mw", "2026-08-11T23:50:00Z", 70, { + ingest_time: "2026-08-12T00:07:00Z", + }); + const otherFeed = obs("region:test:qld", "grid.demand.mw", "2026-08-12T00:05:00Z", 6100, { + ingest_time: "2026-08-12T00:08:00Z", + feed_id: "feed:test:b", + }); + store.silver.append([orig, newer, corr, lateArrival, otherFeed]); + + const engine = new StateEngine(); + const count = engine.loadFromSilver(store); + assert.equal(count, 5); + + // Correction won over the equal-event-time original; late arrival did not regress state. + assert.equal(engine.current("gen:test:alpha", "power.output.mw")[0].value, 92); + assert.equal(engine.current("region:test:qld", "grid.demand.mw")[0].value, 6100); + // History spans both silver day-partitions of feed:test:a, event-time sorted. + assert.deepEqual( + engine.history("gen:test:alpha", "power.output.mw").map((o) => o.value), + [70, 80, 90, 92], + ); + + // Restricting to one feed replays only that feed. + const scoped = new StateEngine(); + assert.equal(scoped.loadFromSilver(store, ["feed:test:b"]), 1); + assert.equal(scoped.current("gen:test:alpha", "power.output.mw").length, 0); +}); diff --git a/intelligence/packages/state/test/windows.test.mjs b/intelligence/packages/state/test/windows.test.mjs new file mode 100644 index 0000000..8193ec3 --- /dev/null +++ b/intelligence/packages/state/test/windows.test.mjs @@ -0,0 +1,127 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { StateEngine, WindowAggregator, WINDOW_5M, WINDOW_1H } from "../dist/index.js"; + +let seq = 0; +const obs = (eventTime, value, extra = {}) => ({ + observation_id: `wobs-${String(++seq).padStart(4, "0")}`, + entity_id: "gen:test:alpha", + metric: "power.output.mw", + value, + unit: "MW", + event_time: eventTime, + ingest_time: eventTime, + source_id: "source:test:feed", + feed_id: "feed:test:a", + quality: "good", + ...extra, +}); + +test("tumbling windows compute min/max/mean/last/count per (entity, metric)", () => { + const agg = new WindowAggregator(); + agg.add(obs("2026-08-12T01:01:00Z", 10)); + agg.add(obs("2026-08-12T01:02:00Z", 30)); + agg.add(obs("2026-08-12T01:04:00Z", 20)); + agg.add(obs("2026-08-12T01:06:00Z", 40)); // next 5-min window + + const w5 = agg.windows("gen:test:alpha", "power.output.mw", WINDOW_5M); + assert.equal(w5.length, 2); + assert.equal(w5[0].window_start, "2026-08-12T01:00:00.000Z"); + assert.equal(w5[0].window_end, "2026-08-12T01:05:00.000Z"); + assert.deepEqual( + [w5[0].min, w5[0].max, w5[0].mean, w5[0].last, w5[0].count], + [10, 30, 20, 20, 3], + ); + assert.deepEqual([w5[1].min, w5[1].max, w5[1].count], [40, 40, 1]); + + // The same observations roll into one 1-hour window. + const w1h = agg.windows("gen:test:alpha", "power.output.mw", WINDOW_1H); + assert.equal(w1h.length, 1); + assert.deepEqual([w1h[0].min, w1h[0].max, w1h[0].mean, w1h[0].last, w1h[0].count], [10, 40, 25, 40, 4]); +}); + +test("last tracks the latest event_time within the window, not arrival order", () => { + const agg = new WindowAggregator(); + agg.add(obs("2026-08-12T01:04:00Z", 99)); + agg.add(obs("2026-08-12T01:01:00Z", 11)); // arrives later, but is earlier in event time + const [w] = agg.windows("gen:test:alpha", "power.output.mw", WINDOW_5M); + assert.equal(w.last, 99); + assert.equal(w.count, 2); +}); + +test("an observation landing in a watermark-closed window patches it and flags revised", () => { + const agg = new WindowAggregator(); // default allowed lateness: 10 min + agg.add(obs("2026-08-12T01:01:00Z", 10)); + agg.add(obs("2026-08-12T01:04:00Z", 30)); + + // Advance the watermark past the [01:00, 01:05) window: 01:59 - 10 min = 01:49. + const advance = agg.add(obs("2026-08-12T01:59:00Z", 50)); + assert.equal(advance.revised, false); + assert.equal(agg.watermarkMs(), Date.parse("2026-08-12T01:49:00Z")); + + // Late data into the closed window: patched, reported as a revision. + const late = agg.add(obs("2026-08-12T01:03:00Z", 100)); + assert.equal(late.accepted, true); + assert.equal(late.revised, true); + const patched5m = late.aggregates.find((a) => a.window_ms === WINDOW_5M); + assert.deepEqual( + [patched5m.min, patched5m.max, patched5m.mean, patched5m.last, patched5m.count], + [10, 100, (10 + 30 + 100) / 3, 30, 3], + ); + + // The revision is visible on subsequent reads too. + const [w] = agg.windows("gen:test:alpha", "power.output.mw", WINDOW_5M); + assert.deepEqual([w.min, w.max, w.count], [10, 100, 3]); + + // The 1h window [01:00, 02:00) is still open (ends after the watermark), + // so the same late observation revised only the 5-minute window. + const patched1h = late.aggregates.find((a) => a.window_ms === WINDOW_1H); + assert.equal(patched1h.count, 4); +}); + +test("allowedLatenessMs is configurable and an observation never closes its own window", () => { + const agg = new WindowAggregator({ allowedLatenessMs: 60_000 }); + assert.equal(agg.watermarkMs(), null); + const first = agg.add(obs("2026-08-12T01:04:00Z", 10)); + assert.equal(first.revised, false); + // Watermark now 01:03; the [01:00, 01:05) window is still open. + const inWindow = agg.add(obs("2026-08-12T01:00:30Z", 20)); + assert.equal(inWindow.revised, false); + // Jump ahead so 01:10 - 1 min lateness closes everything before 01:09... + agg.add(obs("2026-08-12T01:10:00Z", 30)); + // ...then even a same-window repeat of a closed window is a revision. + const late = agg.add(obs("2026-08-12T01:01:00Z", 40)); + assert.equal(late.revised, true); +}); + +test("non-numeric values are skipped by windows but still drive current state", () => { + const engine = new StateEngine(); + const r = engine.ingest([ + obs("2026-08-12T01:00:00Z", { lat: -27.4, lon: 153.1, speed_kn: 11.5 }, { + entity_id: "vessel:imo:9999999", + metric: "vessel.position", + unit: undefined, + }), + ]); + assert.equal(r.deltas.length, 1); + assert.deepEqual(engine.current("vessel:imo:9999999", "vessel.position")[0].value.speed_kn, 11.5); + assert.equal(engine.windows("vessel:imo:9999999", "vessel.position", WINDOW_5M).length, 0); + + const direct = new WindowAggregator(); + const skipped = direct.add(obs("2026-08-12T01:00:00Z", "not-a-number")); + assert.deepEqual(skipped, { accepted: false, revised: false, aggregates: [] }); +}); + +test("engine.windows exposes late revisions after ingest", () => { + const engine = new StateEngine(); + engine.ingest([ + obs("2026-08-12T01:01:00Z", 10), + obs("2026-08-12T01:59:00Z", 50), // watermark → 01:49, closes [01:00, 01:05) + ]); + const late = obs("2026-08-12T01:02:00Z", 90); + const r = engine.ingest([late]); + assert.equal(r.late, 1); // history-only for current state... + const [w] = engine.windows("gen:test:alpha", "power.output.mw", WINDOW_5M); + // ...but the closed 5-min window was patched. + assert.deepEqual([w.min, w.max, w.count], [10, 90, 2]); +}); From ee5ccd5bac54b1644b24847923674b270d6305e6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 01:47:23 +0000 Subject: [PATCH 07/16] =?UTF-8?q?feat(intelligence):=20pact-bridge=20?= =?UTF-8?q?=E2=80=94=20fact=20proposals=20+=20competing-hypotheses=20demo?= =?UTF-8?q?=20(phase=2017)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit InferenceRecord → PACT fact-type proposal mapping (lossless degradation per the RFC: evidence ∪ contrary_evidence verbatim with roles), zero-dep PACT client, and the Gladstone worked example: two agents propose competing cargo claims for a synthetic vessel at RG Tanna, consensus resolves via approve/object, outcomes fold back as CORROBORATED/CONTESTED records that retain full contrary evidence. E2e test boots the reference server. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014yVGqBtvM1zNAd9tvUVNzm --- .../packages/pact-bridge/package.json | 4 +- .../packages/pact-bridge/src/client.ts | 183 ++++++++++++++ .../packages/pact-bridge/src/fact-mapper.ts | 81 ++++++ .../pact-bridge/src/hypotheses-demo.ts | 192 ++++++++++++++ .../packages/pact-bridge/src/index.ts | 24 +- .../pact-bridge/test/hypotheses.test.mjs | 235 ++++++++++++++++++ 6 files changed, 715 insertions(+), 4 deletions(-) create mode 100644 intelligence/packages/pact-bridge/src/client.ts create mode 100644 intelligence/packages/pact-bridge/src/fact-mapper.ts create mode 100644 intelligence/packages/pact-bridge/src/hypotheses-demo.ts create mode 100644 intelligence/packages/pact-bridge/test/hypotheses.test.mjs diff --git a/intelligence/packages/pact-bridge/package.json b/intelligence/packages/pact-bridge/package.json index cd00849..2404b2f 100644 --- a/intelligence/packages/pact-bridge/package.json +++ b/intelligence/packages/pact-bridge/package.json @@ -13,10 +13,10 @@ "node": ">=20" }, "scripts": { - "build": "tsc -p tsconfig.json" + "build": "tsc -p tsconfig.json", + "test": "npm run build && node --test \"test/*.test.mjs\"" }, "dependencies": { - "@pact-tailor/intel": "*", "@pact-tailor/ontology": "*" } } diff --git a/intelligence/packages/pact-bridge/src/client.ts b/intelligence/packages/pact-bridge/src/client.ts new file mode 100644 index 0000000..ee911af --- /dev/null +++ b/intelligence/packages/pact-bridge/src/client.ts @@ -0,0 +1,183 @@ +// PactClient — minimal zero-dependency HTTP client for the PACT reference +// server (reference-server/src/server.ts), built on the global fetch. +// +// Covers exactly the surface the intelligence bridge needs: join a fabric, +// create a `fact`-type proposal, vote (approve/object), read fabric status, +// and reset the server's in-memory fixtures. Every member-scoped call +// asserts the caller via the `X-Pact-Principal` header. + +/** Error thrown for transport failures and non-2xx responses. */ +export class PactClientError extends Error { + constructor( + message: string, + /** HTTP status code, when a response was received. */ + readonly status?: number, + /** Parsed response body, when one was received. */ + readonly body?: unknown, + ) { + super(message); + this.name = "PactClientError"; + } +} + +/** Result of joining a fabric via POST /join-token. */ +export interface JoinResult { + /** `did:key:{registrationId}` — the member's principal_id on the fabric. */ + principalId: string; + apiKey: string; +} + +export interface ProposeInput { + /** Caller-supplied deterministic id (idempotent on re-POST). Optional. */ + proposalId?: string; + sectionId: string; + summary: string; + due_by?: string; +} + +export interface ProposeResult { + proposalId: string; + status: string; + created: boolean; + proposer: string; + required_voters: string[]; + [key: string]: unknown; +} + +export interface VoteResult { + proposalId: string; + recorded: boolean; + decision: string; + [key: string]: unknown; +} + +/** GET /_status snapshot (§4.4.1) — the fields the bridge relies on. */ +export interface FabricStatus { + fabric_id: string; + phase: string; + open_proposals: number; + members: Array>; + pending_obligations: Array>; + [key: string]: unknown; +} + +const DEFAULT_TIMEOUT_MS = 10_000; + +export class PactClient { + private readonly baseUrl: string; + private readonly timeoutMs: number; + + constructor(baseUrl: string, options: { timeoutMs?: number } = {}) { + this.baseUrl = baseUrl.replace(/\/+$/, ""); + this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + } + + /** POST /api/pact/{fabricId}/join-token — anonymous BYOK join (§4.1/§7.1). */ + async join(fabricId: string, agentName: string): Promise { + const res = await this.request<{ registrationId: string; apiKey: string }>( + "POST", + `/api/pact/${encodeURIComponent(fabricId)}/join-token`, + { body: { agentName } }, + ); + if (typeof res.registrationId !== "string" || res.registrationId.length === 0) { + throw new PactClientError( + `join(${fabricId}, ${agentName}): server response missing registrationId`, + undefined, + res, + ); + } + // The created member's principal_id is did:key:{registrationId}. + return { principalId: `did:key:${res.registrationId}`, apiKey: res.apiKey }; + } + + /** POST /api/pact/{fabricId}/proposals — create an open proposal (§4.3/§7.1). */ + async propose( + fabricId: string, + principalId: string, + input: ProposeInput, + ): Promise { + return this.request( + "POST", + `/api/pact/${encodeURIComponent(fabricId)}/proposals`, + { principal: principalId, body: input }, + ); + } + + /** + * POST /api/pact/{fabricId}/proposals/{id}/{verb} — vote on a proposal, + * discharging the caller's §6.5 vote obligation. `approve` resolves the + * proposal approved; `object` blocks it (rejected, §10.5). + */ + async vote( + fabricId: string, + principalId: string, + proposalId: string, + verb: "approve" | "object", + ): Promise { + return this.request( + "POST", + `/api/pact/${encodeURIComponent(fabricId)}/proposals/${encodeURIComponent(proposalId)}/${verb}`, + { principal: principalId, body: {} }, + ); + } + + /** GET /api/pact/{fabricId}/_status — whole-fabric snapshot (§4.4.1). */ + async status(fabricId: string): Promise { + return this.request( + "GET", + `/api/pact/${encodeURIComponent(fabricId)}/_status`, + ); + } + + /** POST /__reset — reference-server test helper: reseed in-memory fixtures. */ + async reset(): Promise { + await this.request<{ reset: boolean }>("POST", "/__reset"); + } + + private async request( + method: "GET" | "POST", + path: string, + opts: { principal?: string; body?: unknown } = {}, + ): Promise { + const url = `${this.baseUrl}${path}`; + const headers: Record = {}; + if (opts.principal !== undefined) headers["X-Pact-Principal"] = opts.principal; + let bodyText: string | undefined; + if (opts.body !== undefined) { + headers["Content-Type"] = "application/json"; + bodyText = JSON.stringify(opts.body); + } + + let res: Response; + try { + res = await fetch(url, { + method, + headers, + body: bodyText, + signal: AbortSignal.timeout(this.timeoutMs), + }); + } catch (err) { + const cause = err instanceof Error ? `${err.name}: ${err.message}` : String(err); + throw new PactClientError( + `PactClient: ${method} ${url} failed (${cause}). Is the reference server running?`, + ); + } + + const text = await res.text(); + let parsed: unknown; + try { + parsed = text === "" ? undefined : JSON.parse(text); + } catch { + parsed = text; + } + + if (!res.ok) { + throw new PactClientError( + `PactClient: ${method} ${url} → HTTP ${res.status}: ${text.slice(0, 500)}`, + res.status, + parsed, + ); + } + return parsed as T; + } +} diff --git a/intelligence/packages/pact-bridge/src/fact-mapper.ts b/intelligence/packages/pact-bridge/src/fact-mapper.ts new file mode 100644 index 0000000..463d3bc --- /dev/null +++ b/intelligence/packages/pact-bridge/src/fact-mapper.ts @@ -0,0 +1,81 @@ +// fact-mapper — InferenceRecord ⇄ PACT `fact`-type proposal mapping. +// +// Implements the total, lossless-degradation mapping of the RFC +// (docs/v2-prep/rfc-intelligence-fact-evidence-extension.md §3): an +// `au.tailor.intel.claim` inference envelope degrades to the built-in +// `fact` proposal payload { claim, evidence, tier, sources } with nothing +// dropped — evidence and contrary evidence travel together, each item +// keeping its `role`, so counter-evidence survives the round trip. + +import type { EvidenceRef, InferenceRecord } from "@pact-tailor/ontology"; + +/** A PACT proposal carrying a `fact`-type payload derived from an inference. */ +export interface FactProposal { + /** Field grammar of the fact type: `claim:{inference_id}`. */ + sectionId: string; + /** The human-readable claim sentence plus its confidence. */ + summary: string; + /** Built-in `fact` proposal payload — { claim, evidence, tier, sources }. */ + payload: { + claim: string; + evidence: unknown[]; + tier: string; + sources: string[]; + }; +} + +/** + * Map an InferenceRecord to a built-in `fact`-type proposal (RFC §3). + * + * - `sectionId` = `claim:${inference_id}` (the fact type's field grammar). + * - `payload.evidence` = evidence ∪ contrary_evidence, verbatim — every item + * keeps its `role` ("supports" | "contradicts") so contrary evidence is + * never lost in the degradation. + * - `summary` = the claim sentence + confidence. + * + * The mapping is total: every inference maps; nothing epistemically + * significant is dropped. + */ +export function toFactProposal(inference: InferenceRecord): FactProposal { + const evidence: EvidenceRef[] = [ + ...inference.evidence.map((e) => ({ ...e })), + ...(inference.contrary_evidence ?? []).map((e) => ({ ...e })), + ]; + return { + sectionId: `claim:${inference.inference_id}`, + summary: `${inference.claim} (confidence ${inference.confidence})`, + payload: { + claim: inference.claim, + evidence, + tier: inference.tier, + sources: [...inference.sources], + }, + }; +} + +/** + * Fold a PACT proposal outcome back into the inference lifecycle. + * + * Returns a NEW record (append-only stance — the input is never mutated; + * a status change is a new revision, never an in-place edit that loses the + * contested history): + * + * - `"approved"` → status `CORROBORATED` + * - `"rejected"` → status `CONTESTED` + * + * The PACT backref (`{ fabric_id, proposal_id }`) is filled from the `pact` + * argument, merged over any backref already on the record. All evidence and + * contrary-evidence arrays are retained verbatim. + */ +export function fromProposalOutcome( + inference: InferenceRecord, + outcome: "approved" | "rejected", + pact?: { fabric_id?: string; proposal_id?: string }, +): InferenceRecord { + const next = structuredClone(inference); + next.status = outcome === "approved" ? "CORROBORATED" : "CONTESTED"; + if (pact !== undefined || (inference.pact !== undefined && inference.pact !== null)) { + next.pact = { ...(inference.pact ?? {}), ...(pact ?? {}) }; + } + return next; +} diff --git a/intelligence/packages/pact-bridge/src/hypotheses-demo.ts b/intelligence/packages/pact-bridge/src/hypotheses-demo.ts new file mode 100644 index 0000000..391cd1d --- /dev/null +++ b/intelligence/packages/pact-bridge/src/hypotheses-demo.ts @@ -0,0 +1,192 @@ +// The Gladstone competing-hypotheses demo (RFC §5, implemented). +// +// A vessel is loading at Gladstone's RG Tanna coal terminal. Two rule agents +// derive competing cargo hypotheses for the same vessel: +// +// Agent A ("cargo-inference-a") — metallurgical coal, confidence 0.91, +// citing AIS berth-arrival + draught-change observations and a structural +// terminal-supply edge. +// Agent B ("schedule-checker-b") — thermal coal, confidence 0.34, citing a +// contrary shipping-schedule observation; its contrary_evidence cites the +// met-coal hypothesis' own supporting observation and the met-coal +// inference itself. +// +// Both publish into a fresh PACT fabric as `fact`-type proposals. A third +// agent ("resolver-c") approves A's proposal; A objects to B's (a blocking +// objection rejects it per §10.5). The outcomes fold back into the inference +// lifecycle: A's record becomes CORROBORATED, B's becomes CONTESTED — both as +// NEW records (append-only) retaining every evidence and contrary-evidence +// item, with the PACT backref filled. +// +// All data is synthetic and clearly marked so: vessel:imo:9000001 is not a +// real ship, timestamps are fixed constants. + +import { createValidators, type InferenceRecord } from "@pact-tailor/ontology"; +import { toFactProposal, fromProposalOutcome } from "./fact-mapper.js"; +import { PactClient, type FabricStatus } from "./client.js"; + +export interface DemoResult { + /** Agent A's met-coal inference after consensus — status CORROBORATED. */ + corroborated: InferenceRecord; + /** Agent B's thermal-coal inference after consensus — status CONTESTED. */ + contested: InferenceRecord; + /** Final fabric _status snapshot (0 open proposals once resolved). */ + fabricStatus: FabricStatus; +} + +// Fixed synthetic constants — deterministic records, clearly not real data. +const PRODUCED_AT = "2026-08-12T03:15:00Z"; +const EVENT_FROM = "2026-08-12T01:00:00Z"; +const EVENT_TO = "2026-08-12T03:10:00Z"; +const VESSEL = "vessel:imo:9000001"; +const TERMINAL = "terminal:au-qld:rg-tanna"; + +const MET_COAL_ID = "infer:2026-08-12:gladstone-met-coal"; +const THERMAL_ID = "infer:2026-08-12:gladstone-thermal-coal"; + +// Synthetic evidence refs shared between the two hypotheses. +const OBS_BERTH_ARRIVAL = "obs:demo:ais-berth-arrival-rg-tanna-b1"; +const OBS_DRAUGHT_CHANGE = "obs:demo:draught-increase-9000001"; +const REL_SUPPLY_EDGE = "rel:rg-tanna-supplies-met-coal"; +const OBS_SCHEDULE_THERMAL = "obs:demo:shipping-schedule-thermal-9000001"; + +/** + * Build the two competing InferenceRecords, both status INFERRED. + * Exported so tests can exercise the mapper against the same fixtures. + */ +export function buildGladstoneHypotheses(): { + metCoal: InferenceRecord; + thermal: InferenceRecord; +} { + const metCoal: InferenceRecord = { + inference_id: MET_COAL_ID, + claim: + "Vessel IMO 9000001, loading at RG Tanna terminal (Gladstone), is loading metallurgical coal.", + claim_structured: { + subject: VESSEL, + predicate: "carries", + object: "commodity:coal-metallurgical", + qualifiers: { loading_at: TERMINAL, synthetic: true }, + }, + tier: "B", + confidence: 0.91, + status: "INFERRED", + method: "rule:cargo-inference@1.0.0", + evidence: [ + { kind: "observation", ref: OBS_BERTH_ARRIVAL, role: "supports" }, + { kind: "observation", ref: OBS_DRAUGHT_CHANGE, role: "supports" }, + { kind: "structural", ref: REL_SUPPLY_EDGE, role: "supports" }, + ], + contrary_evidence: [ + { kind: "observation", ref: OBS_SCHEDULE_THERMAL, role: "contradicts" }, + ], + sources: ["source:ais:aisstream", "source:demo:gladstone-berth-registry"], + produced_at: PRODUCED_AT, + event_time_range: { from: EVENT_FROM, to: EVENT_TO }, + lineage: { + correlation_id: "corr-gladstone-demo", + in_response_to: null, + prev_hash: null, + }, + notes: "Synthetic demo record — vessel:imo:9000001 is not a real ship.", + }; + + const thermal: InferenceRecord = { + inference_id: THERMAL_ID, + claim: + "Vessel IMO 9000001, loading at RG Tanna terminal (Gladstone), is loading thermal coal.", + claim_structured: { + subject: VESSEL, + predicate: "carries", + object: "commodity:coal-thermal", + qualifiers: { loading_at: TERMINAL, synthetic: true }, + }, + tier: "C", + confidence: 0.34, + status: "INFERRED", + method: "rule:cargo-schedule-check@1.0.0", + evidence: [ + { kind: "observation", ref: OBS_SCHEDULE_THERMAL, role: "supports" }, + ], + contrary_evidence: [ + { kind: "observation", ref: OBS_BERTH_ARRIVAL, role: "contradicts" }, + { kind: "inference", ref: MET_COAL_ID, role: "contradicts" }, + ], + sources: ["source:demo:gladstone-shipping-schedule"], + produced_at: PRODUCED_AT, + event_time_range: { from: EVENT_FROM, to: EVENT_TO }, + lineage: { + correlation_id: "corr-gladstone-demo", + in_response_to: null, + prev_hash: null, + }, + notes: "Synthetic demo record — vessel:imo:9000001 is not a real ship.", + }; + + return { metCoal, thermal }; +} + +/** + * Run the full competing-hypotheses flow against a PACT reference server at + * `baseUrl`. Returns the two post-consensus inference records (both validated + * against the inference JSON Schema) plus the final fabric status. + */ +export async function runGladstoneDemo(baseUrl: string): Promise { + const client = new PactClient(baseUrl); + const suffix = Math.random().toString(36).slice(2, 10); + const fabricId = `intel-gladstone-demo-${suffix}`; + + // Three agents join a fresh fabric. + const agentA = await client.join(fabricId, "cargo-inference-a"); + const agentB = await client.join(fabricId, "schedule-checker-b"); + const agentC = await client.join(fabricId, "resolver-c"); + + const { metCoal, thermal } = buildGladstoneHypotheses(); + + // Each hypothesis is published as a `fact`-type proposal (RFC §3 mapping). + const factA = toFactProposal(metCoal); + const factB = toFactProposal(thermal); + + const proposalA = await client.propose(fabricId, agentA.principalId, { + proposalId: `prop-met-coal-${suffix}`, + sectionId: factA.sectionId, + summary: factA.summary, + }); + const proposalB = await client.propose(fabricId, agentB.principalId, { + proposalId: `prop-thermal-coal-${suffix}`, + sectionId: factB.sectionId, + summary: factB.summary, + }); + + // Consensus: C (holding a vote obligation on A's proposal) approves it; + // A objects to B's proposal — a blocking objection rejects it (§10.5). + await client.vote(fabricId, agentC.principalId, proposalA.proposalId, "approve"); + await client.vote(fabricId, agentA.principalId, proposalB.proposalId, "object"); + + // Fold the outcomes back into the inference lifecycle as NEW records. + const corroborated = fromProposalOutcome(metCoal, "approved", { + fabric_id: fabricId, + proposal_id: proposalA.proposalId, + }); + const contested = fromProposalOutcome(thermal, "rejected", { + fabric_id: fabricId, + proposal_id: proposalB.proposalId, + }); + + // Both final records MUST validate against the inference JSON Schema. + const validators = createValidators(); + for (const [label, record] of [ + ["corroborated", corroborated], + ["contested", contested], + ] as const) { + const result = validators.inference(record); + if (!result.ok) { + throw new Error( + `gladstone demo produced a schema-invalid ${label} inference: ${result.errors.join("; ")}`, + ); + } + } + + const fabricStatus = await client.status(fabricId); + return { corroborated, contested, fabricStatus }; +} diff --git a/intelligence/packages/pact-bridge/src/index.ts b/intelligence/packages/pact-bridge/src/index.ts index c26c923..a840b93 100644 --- a/intelligence/packages/pact-bridge/src/index.ts +++ b/intelligence/packages/pact-bridge/src/index.ts @@ -1,2 +1,22 @@ -// @pact-tailor/pact-bridge — inference → PACT fact proposals. Populated in the pact-bridge commit. -export {}; +// @pact-tailor/pact-bridge — inference → PACT fact proposals. +// +// Bridges class-C derived intelligence (InferenceRecord) into a PACT fabric: +// inferences publish as `fact`-type proposals, and competing hypotheses +// resolve through PACT's propose/object machinery. See +// docs/v2-prep/rfc-intelligence-fact-evidence-extension.md. + +export { toFactProposal, fromProposalOutcome, type FactProposal } from "./fact-mapper.js"; +export { + PactClient, + PactClientError, + type JoinResult, + type ProposeInput, + type ProposeResult, + type VoteResult, + type FabricStatus, +} from "./client.js"; +export { + runGladstoneDemo, + buildGladstoneHypotheses, + type DemoResult, +} from "./hypotheses-demo.js"; diff --git a/intelligence/packages/pact-bridge/test/hypotheses.test.mjs b/intelligence/packages/pact-bridge/test/hypotheses.test.mjs new file mode 100644 index 0000000..3fb9694 --- /dev/null +++ b/intelligence/packages/pact-bridge/test/hypotheses.test.mjs @@ -0,0 +1,235 @@ +// pact-bridge tests: unit tests for the fact mapper plus the end-to-end +// Gladstone competing-hypotheses demo against a locally spawned reference +// server (reference-server/dist/server.js). + +import test from "node:test"; +import assert from "node:assert/strict"; +import { spawn, spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import path from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; +import { createValidators } from "@pact-tailor/ontology"; +import { + toFactProposal, + fromProposalOutcome, + buildGladstoneHypotheses, + runGladstoneDemo, +} from "../dist/index.js"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const refServerDir = path.resolve(here, "../../../../reference-server"); +const serverJs = path.join(refServerDir, "dist", "server.js"); + +// --------------------------------------------------------------------------- +// Unit: toFactProposal +// --------------------------------------------------------------------------- + +test("toFactProposal: sectionId follows the fact field grammar claim:{id}", () => { + const { metCoal } = buildGladstoneHypotheses(); + const fact = toFactProposal(metCoal); + assert.equal(fact.sectionId, `claim:${metCoal.inference_id}`); + assert.match(fact.sectionId, /^claim:infer:[0-9]{4}-[0-9]{2}-[0-9]{2}:[a-z0-9][a-z0-9-]*$/); +}); + +test("toFactProposal: evidence union keeps every item and its role", () => { + const { thermal } = buildGladstoneHypotheses(); + const fact = toFactProposal(thermal); + + // union = evidence ∪ contrary_evidence, verbatim, supports first. + assert.deepEqual(fact.payload.evidence, [ + ...thermal.evidence, + ...thermal.contrary_evidence, + ]); + assert.equal( + fact.payload.evidence.length, + thermal.evidence.length + thermal.contrary_evidence.length, + ); + for (const item of fact.payload.evidence) { + assert.ok( + item.role === "supports" || item.role === "contradicts", + `evidence item lost its role: ${JSON.stringify(item)}`, + ); + } + // contrary evidence survives the degradation with role intact. + assert.ok( + fact.payload.evidence.some((e) => e.role === "contradicts"), + "contrary evidence must survive the fact mapping", + ); + // rest of the payload is the total RFC §3 mapping. + assert.equal(fact.payload.claim, thermal.claim); + assert.equal(fact.payload.tier, thermal.tier); + assert.deepEqual(fact.payload.sources, thermal.sources); + // summary = claim sentence + confidence. + assert.ok(fact.summary.includes(thermal.claim), "summary carries the claim sentence"); + assert.ok(fact.summary.includes(String(thermal.confidence)), "summary carries the confidence"); +}); + +test("toFactProposal: no contrary_evidence → union is just the supports array", () => { + const { metCoal } = buildGladstoneHypotheses(); + const noContrary = structuredClone(metCoal); + delete noContrary.contrary_evidence; + const fact = toFactProposal(noContrary); + assert.deepEqual(fact.payload.evidence, noContrary.evidence); +}); + +// --------------------------------------------------------------------------- +// Unit: fromProposalOutcome +// --------------------------------------------------------------------------- + +test("fromProposalOutcome: returns a NEW record; the original is untouched", () => { + const { metCoal } = buildGladstoneHypotheses(); + const before = structuredClone(metCoal); + + const next = fromProposalOutcome(metCoal, "approved", { + fabric_id: "fab-test", + proposal_id: "prop-test", + }); + + // New object, deep-copied arrays. + assert.notEqual(next, metCoal); + assert.notEqual(next.evidence, metCoal.evidence); + assert.notEqual(next.contrary_evidence, metCoal.contrary_evidence); + // Original untouched — append-only stance. + assert.deepEqual(metCoal, before); + assert.equal(metCoal.status, "INFERRED"); + assert.equal(metCoal.pact, undefined); + // New record: status transitioned, pact backref filled, evidence retained. + assert.equal(next.status, "CORROBORATED"); + assert.deepEqual(next.pact, { fabric_id: "fab-test", proposal_id: "prop-test" }); + assert.deepEqual(next.evidence, metCoal.evidence); + assert.deepEqual(next.contrary_evidence, metCoal.contrary_evidence); +}); + +test("fromProposalOutcome: rejected → CONTESTED", () => { + const { thermal } = buildGladstoneHypotheses(); + const next = fromProposalOutcome(thermal, "rejected", { + fabric_id: "fab-test", + proposal_id: "prop-test-2", + }); + assert.equal(next.status, "CONTESTED"); + assert.equal(thermal.status, "INFERRED"); + assert.deepEqual(next.contrary_evidence, thermal.contrary_evidence); +}); + +// --------------------------------------------------------------------------- +// e2e: Gladstone demo against a spawned reference server +// --------------------------------------------------------------------------- + +function ensureReferenceServerBuilt() { + if (existsSync(serverJs)) return; + for (const args of [ + ["ci", "--no-audit", "--no-fund"], + ["run", "build"], + ]) { + const r = spawnSync("npm", args, { cwd: refServerDir, stdio: "inherit" }); + assert.equal( + r.status, + 0, + `npm ${args.join(" ")} failed in ${refServerDir} (status ${r.status})`, + ); + } + assert.ok(existsSync(serverJs), `reference server build produced no ${serverJs}`); +} + +/** + * Spawn the reference server on an ephemeral port and wait for /healthz. + * Tries a few random ports in case one is already taken. + * Returns { child, baseUrl }. + */ +async function startReferenceServer() { + let lastError = ""; + for (let attempt = 0; attempt < 3; attempt++) { + const port = 4300 + Math.floor(Math.random() * 1000); + const baseUrl = `http://127.0.0.1:${port}`; + const child = spawn(process.execPath, [serverJs, "--port", String(port)], { + stdio: ["ignore", "pipe", "pipe"], + }); + let output = ""; + child.stdout.on("data", (c) => (output += String(c))); + child.stderr.on("data", (c) => (output += String(c))); + let exited = false; + child.on("exit", () => { + exited = true; + }); + + // Poll /healthz for up to 10s. + const deadline = Date.now() + 10_000; + while (Date.now() < deadline && !exited) { + try { + const res = await fetch(`${baseUrl}/healthz`, { + signal: AbortSignal.timeout(1_000), + }); + if (res.status === 200) return { child, baseUrl }; + } catch { + // not up yet + } + await sleep(100); + } + child.kill(); + lastError = `port ${port}: exited=${exited} output=${output.slice(0, 300)}`; + } + assert.fail(`reference server failed to become healthy: ${lastError}`); +} + +test("gladstone demo e2e: competing hypotheses resolve via propose/object", async () => { + ensureReferenceServerBuilt(); + const { child, baseUrl } = await startReferenceServer(); + try { + const result = await runGladstoneDemo(baseUrl); + const { corroborated, contested, fabricStatus } = result; + + // A's met-coal inference won consensus. + assert.equal(corroborated.status, "CORROBORATED"); + assert.ok(corroborated.pact, "corroborated record must carry the pact backref"); + assert.ok( + typeof corroborated.pact.proposal_id === "string" && + corroborated.pact.proposal_id.length > 0, + "corroborated.pact.proposal_id must be set", + ); + assert.match(String(corroborated.pact.fabric_id), /^intel-gladstone-demo-/); + + // B's thermal inference was objected to. + assert.equal(contested.status, "CONTESTED"); + assert.ok( + typeof contested.pact?.proposal_id === "string" && contested.pact.proposal_id.length > 0, + "contested.pact.proposal_id must be set", + ); + + // The contested record retains BOTH its own supporting evidence and its + // contrary evidence — losing consensus loses nothing epistemic. + assert.ok(contested.evidence.length >= 1, "contested must retain supports evidence"); + for (const e of contested.evidence) assert.equal(e.role, "supports"); + assert.ok( + Array.isArray(contested.contrary_evidence) && contested.contrary_evidence.length >= 1, + "contested must retain contrary_evidence", + ); + for (const e of contested.contrary_evidence) assert.equal(e.role, "contradicts"); + // The winner keeps its contrary evidence too. + assert.ok( + Array.isArray(corroborated.contrary_evidence) && + corroborated.contrary_evidence.length >= 1, + "corroborated must retain contrary_evidence", + ); + + // Both final records are schema-valid inferences. + const validators = createValidators(); + for (const [label, record] of [ + ["corroborated", corroborated], + ["contested", contested], + ]) { + const v = validators.inference(record); + assert.ok( + v.ok, + `${label} record must be schema-valid: ${v.ok ? "" : v.errors.join("; ")}`, + ); + } + + // The fabric converged: both proposals resolved, none left open. + assert.equal(fabricStatus.open_proposals, 0, "fabric must show 0 open proposals"); + assert.equal(fabricStatus.fabric_id, corroborated.pact.fabric_id); + assert.equal(fabricStatus.members.length, 3, "three agents joined the fabric"); + } finally { + child.kill(); + } +}); From b648c33c855f5cbbf4af8ae11a79611ebf70f79e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 01:52:40 +0000 Subject: [PATCH 08/16] =?UTF-8?q?feat(intelligence):=20global=20grid=20reg?= =?UTF-8?q?istry=20=E2=80=94=2071=20electrical=20systems=20(phase=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Electrical systems as the organising unit, countries as metadata: synchronous areas, interconnections, ISO/RTOs, TSO areas, national/regional/isolated grids with PART_OF hierarchy, operators, market structure, interconnections, and a live-data availability rating per system. Verified via research: Japan's 50/60 Hz split, Hydro-Québec as its own interconnection, Ukraine and Baltic synchronisation to Continental Europe, NWIS isolation. Grid market subfields made nullable (null = honestly unknown). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014yVGqBtvM1zNAd9tvUVNzm --- .../registries/grids/africa-middle-east.yaml | 95 ++++ .../data/registries/grids/americas-other.yaml | 100 ++++ intelligence/data/registries/grids/asia.yaml | 301 +++++++++++ intelligence/data/registries/grids/au.yaml | 229 ++++++++ .../data/registries/grids/europe.yaml | 493 ++++++++++++++++++ .../data/registries/grids/isolated.yaml | 161 ++++++ .../data/registries/grids/north-america.yaml | 333 ++++++++++++ intelligence/schemas/grid.schema.json | 138 ++++- 8 files changed, 1822 insertions(+), 28 deletions(-) create mode 100644 intelligence/data/registries/grids/africa-middle-east.yaml create mode 100644 intelligence/data/registries/grids/americas-other.yaml create mode 100644 intelligence/data/registries/grids/asia.yaml create mode 100644 intelligence/data/registries/grids/au.yaml create mode 100644 intelligence/data/registries/grids/europe.yaml create mode 100644 intelligence/data/registries/grids/isolated.yaml create mode 100644 intelligence/data/registries/grids/north-america.yaml diff --git a/intelligence/data/registries/grids/africa-middle-east.yaml b/intelligence/data/registries/grids/africa-middle-east.yaml new file mode 100644 index 0000000..5e63c93 --- /dev/null +++ b/intelligence/data/registries/grids/africa-middle-east.yaml @@ -0,0 +1,95 @@ +# Grid registry — Africa & Middle East. Class-A structural records. See ../../schemas/grid.schema.json +records: + - grid_id: grid:za + name: South African National Grid (Eskom) + kind: national_grid + parent_grid: null + countries: [za] + frequency_hz: 50 + timezone: Africa/Johannesburg + operators: + - { name: National Transmission Company South Africa (NTCSA), role: system_operator } + - { name: National Transmission Company South Africa (NTCSA), role: transmission_owner } + - { name: NERSA, role: regulator } + market: + structure: vertically integrated single buyer; unbundling under way with NTCSA carved out of Eskom in 2024 + price_mechanism: regulated tariffs; wheeling and trading framework emerging + dispatch_interval: null + interconnections: [] + live_data: + availability: partial + notes: hourly system data (demand, generation by source, loadshedding) on the Eskom data portal with short delay + source_ids: [source:eskom:data-portal] + demand_range_mw: { min: 19000, max: 34000 } + sources: [source:eskom:data-portal] + notes: "Anchor of the Southern African Power Pool with synchronous ties to Mozambique, Botswana, Zimbabwe, Namibia, Eswatini and beyond; HVDC import from Cahora Bassa; coal-dominated, loadshedding-prone." + + - grid_id: grid:gccia + name: GCC Interconnection + kind: interconnection + parent_grid: null + countries: [kw, sa, bh, qa, ae, om] + frequency_hz: 50 + timezone: Asia/Riyadh + operators: + - { name: Gulf Cooperation Council Interconnection Authority (GCCIA), role: system_operator } + market: + structure: primarily reserve sharing and emergency support; regional energy trading platform developing + price_mechanism: bilateral and scheduled exchanges + dispatch_interval: null + interconnections: [] + live_data: + availability: minimal + notes: periodic GCCIA reports only; no public operational feed + source_ids: [source:wikidata:sparql] + demand_range_mw: null + sources: [source:wikidata:sparql] + notes: "400 kV 50 Hz spine linking Kuwait, Bahrain, Qatar, UAE and Oman; Saudi Arabia's 60 Hz system connects via an 1800 MW back-to-back HVDC at Al Fadhili; first-phase link to southern Iraq energized 2024." + + - grid_id: grid:eg + name: Egyptian National Grid + kind: national_grid + parent_grid: null + countries: [eg] + frequency_hz: 50 + timezone: Africa/Cairo + operators: + - { name: Egyptian Electricity Transmission Company (EETC), role: tso } + - { name: Egyptian Electricity Transmission Company (EETC), role: transmission_owner } + - { name: EgyptERA, role: regulator } + market: + structure: single buyer (EETC) under gradual liberalization + price_mechanism: regulated tariffs + dispatch_interval: null + interconnections: [] + live_data: + availability: minimal + notes: annual and periodic reports from EEHC and the ministry; no public operational feed + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 20000, max: 38000 } + sources: [source:wikidata:sparql] + notes: "AC ties to Libya and Jordan (legacy Eight Country Interconnection); 3000 MW HVDC to Saudi Arabia under construction (first stage mid-2020s); gas-dominated with Benban solar complex." + + - grid_id: grid:ng + name: Nigerian National Grid + kind: national_grid + parent_grid: null + countries: [ng] + frequency_hz: 50 + timezone: Africa/Lagos + operators: + - { name: Nigerian Independent System Operator (NISO), role: system_operator } + - { name: Transmission Company of Nigeria (TCN), role: transmission_owner } + - { name: Nigerian Electricity Regulatory Commission, role: regulator } + market: + structure: transitional electricity market; NBET-legacy and bilateral contracts; state markets emerging under the 2023 Electricity Act + price_mechanism: regulated multi-year tariff orders + dispatch_interval: null + interconnections: [] + live_data: + availability: partial + notes: daily operational reports with hourly generation from the system operator; frequent grid-event reporting + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 3500, max: 5800 } + sources: [source:wikidata:sparql] + notes: "Grid delivers ~5 GW against ~13 GW installed and far larger latent demand; repeated system collapses; WAPP member with ties to Benin and Niger; NISO carved out of TCN in 2025." diff --git a/intelligence/data/registries/grids/americas-other.yaml b/intelligence/data/registries/grids/americas-other.yaml new file mode 100644 index 0000000..39ce62b --- /dev/null +++ b/intelligence/data/registries/grids/americas-other.yaml @@ -0,0 +1,100 @@ +# Grid registry — Americas (other). Class-A structural records. See ../../schemas/grid.schema.json +records: + - grid_id: grid:br-sin + name: Sistema Interligado Nacional (Brazil) + kind: national_grid + parent_grid: null + countries: [br] + frequency_hz: 60 + timezone: America/Sao_Paulo + operators: + - { name: Operador Nacional do Sistema Eletrico (ONS), role: system_operator } + - { name: Camara de Comercializacao de Energia Eletrica (CCEE), role: market_operator } + - { name: ANEEL, role: regulator } + market: + structure: cost-based centralized dispatch; regulated and free contracting environments + price_mechanism: hourly settlement price (PLD) derived from dispatch models + dispatch_interval: null + interconnections: [] + live_data: + availability: rich + notes: ONS open-data platform plus real-time operation dashboards; CCEE market data + source_ids: [source:ons-brazil:open-data] + demand_range_mw: { min: 55000, max: 103000 } + sources: [source:ons-brazil:open-data] + notes: "Continental-scale hydro-dominated system in four subsystems (SE/CO, S, NE, N); Itaipu shared with Paraguay; asynchronous frequency-converter ties to Argentina and Uruguay; isolated systems remain in parts of the Amazon." + + - grid_id: grid:ar-sadi + name: Sistema Argentino de Interconexion (SADI) + kind: national_grid + parent_grid: null + countries: [ar] + frequency_hz: 50 + timezone: America/Argentina/Buenos_Aires + operators: + - { name: CAMMESA, role: system_operator } + - { name: CAMMESA, role: market_operator } + - { name: Transener, role: transmission_owner } + - { name: ENRE, role: regulator } + market: + structure: cost-based wholesale market (MEM) administered by CAMMESA; heavy price regulation + price_mechanism: audited-cost dispatch with regulated seasonal prices + dispatch_interval: null + interconnections: [] + live_data: + availability: partial + notes: CAMMESA publishes demand, generation and market data with modest delay; some near-real-time aggregates + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 12000, max: 30000 } + sources: [source:wikidata:sparql] + notes: "50 Hz system with synchronous ties to Uruguay and Paraguay (Salto Grande, Yacyreta); asynchronous frequency-converter links to Brazil; minor rarely-used tie to Chile at Salta." + + - grid_id: grid:cl-sen + name: Sistema Electrico Nacional (Chile) + kind: national_grid + parent_grid: null + countries: [cl] + frequency_hz: 50 + timezone: America/Santiago + operators: + - { name: Coordinador Electrico Nacional, role: system_operator } + - { name: Transelec, role: transmission_owner } + - { name: Comision Nacional de Energia, role: regulator } + market: + structure: mandatory cost-based pool; no bid-based market; contracts via auctions + price_mechanism: audited-cost marginal pricing at system nodes + dispatch_interval: null + interconnections: [] + live_data: + availability: partial + notes: extensive open-data platform from the Coordinador with hourly operational data and some real-time aggregates + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 7000, max: 12000 } + sources: [source:wikidata:sparql] + notes: "Formed 2017 by merging the SIC and SING; ~3100 km long and thin; world-class Atacama solar driving big daily flows; Aysen and Magallanes systems remain separate; small tie to Argentina." + + - grid_id: grid:siepac + name: SIEPAC / Central American Interconnection + kind: interconnection + parent_grid: null + countries: [gt, sv, hn, ni, cr, pa] + frequency_hz: 60 + timezone: America/Guatemala + operators: + - { name: Ente Operador Regional (EOR), role: system_operator } + - { name: Ente Operador Regional (EOR), role: market_operator } + - { name: Empresa Propietaria de la Red (EPR), role: transmission_owner } + - { name: CRIE, role: regulator } + market: + structure: regional electricity market (MER) layered over six national markets + price_mechanism: regional nodal prices for cross-border transactions + dispatch_interval: null + interconnections: + - { to_grid: grid:mx-sin, name: "AC tie Mexico-Guatemala, normally operated radially", capacity_mw: 200 } + live_data: + availability: minimal + notes: EOR publishes regional operational and market reports; no public real-time feed + source_ids: [source:wikidata:sparql] + demand_range_mw: null + sources: [source:wikidata:sparql] + notes: "1800 km 230 kV backbone (~300 MW design transfer) linking six synchronously operated national grids from Guatemala to Panama; Colombia link long planned; listed here rather than under Africa/Middle East despite the tasking bullet." diff --git a/intelligence/data/registries/grids/asia.yaml b/intelligence/data/registries/grids/asia.yaml new file mode 100644 index 0000000..d476fc3 --- /dev/null +++ b/intelligence/data/registries/grids/asia.yaml @@ -0,0 +1,301 @@ +# Grid registry — Asia. Class-A structural records. See ../../schemas/grid.schema.json +records: + - grid_id: grid:jp-east + name: Eastern Japan 50 Hz Area + kind: synchronous_area + parent_grid: null + countries: [jp] + frequency_hz: 50 + timezone: Asia/Tokyo + operators: + - { name: "Organization for Cross-regional Coordination of Transmission Operators (OCCTO)", role: system_operator } + - { name: TEPCO Power Grid, role: tso } + - { name: Tohoku Electric Power Network, role: tso } + - { name: Hokkaido Electric Power Network, role: tso } + - { name: Japan Electric Power Exchange (JEPX), role: market_operator } + market: + structure: liberalized retail; JEPX day-ahead/intraday spot with area prices; TSO balancing markets + price_mechanism: zonal (area) marginal prices on JEPX, 30-minute products + dispatch_interval: null + interconnections: + - { to_grid: grid:jp-west, name: "frequency converter stations (Sakuma, Shin-Shinano, Higashi-Shimizu, Minami-Fukumitsu)", capacity_mw: 2100 } + live_data: + availability: partial + notes: real-time area demand and hourly generation mix published by TSOs and OCCTO; JEPX prices public; no unit-level dispatch API + source_ids: [source:occto:web] + demand_range_mw: null + sources: [source:occto:web] + notes: "Tokyo, Tohoku and Hokkaido areas; Hokkaido is joined to Honshu by HVDC within the 50 Hz half; the 50/60 Hz split dates to 1890s generator imports; converter capacity being expanded toward 3 GW under OCCTO's plan." + + - grid_id: grid:jp-west + name: Western Japan 60 Hz Area + kind: synchronous_area + parent_grid: null + countries: [jp] + frequency_hz: 60 + timezone: Asia/Tokyo + operators: + - { name: "Organization for Cross-regional Coordination of Transmission Operators (OCCTO)", role: system_operator } + - { name: Kansai Transmission and Distribution, role: tso } + - { name: Chubu Electric Power Grid, role: tso } + - { name: Kyushu Electric Power Transmission and Distribution, role: tso } + - { name: Japan Electric Power Exchange (JEPX), role: market_operator } + market: + structure: liberalized retail; JEPX day-ahead/intraday spot with area prices; TSO balancing markets + price_mechanism: zonal (area) marginal prices on JEPX, 30-minute products + dispatch_interval: null + interconnections: + - { to_grid: grid:jp-east, name: "frequency converter stations (Sakuma, Shin-Shinano, Higashi-Shimizu, Minami-Fukumitsu)", capacity_mw: 2100 } + live_data: + availability: partial + notes: real-time area demand and hourly generation mix published by TSOs and OCCTO; JEPX prices public + source_ids: [source:occto:web] + demand_range_mw: null + sources: [source:occto:web] + notes: "Comprises the Chubu, Hokuriku, Kansai, Chugoku, Shikoku and Kyushu TSO areas; Okinawa is a separate isolated 60 Hz island system outside both synchronous areas." + + - grid_id: grid:cn-sgcc + name: State Grid Corporation of China Area + kind: national_grid + parent_grid: null + countries: [cn] + frequency_hz: 50 + timezone: Asia/Shanghai + operators: + - { name: State Grid Corporation of China, role: system_operator } + - { name: State Grid Corporation of China, role: transmission_owner } + - { name: National Energy Administration, role: regulator } + market: + structure: medium/long-term contracts dominant; provincial spot markets rolling out; unified national market planned + price_mechanism: provincial spot prices where operating; regulated tariffs elsewhere + dispatch_interval: null + interconnections: [] + live_data: + availability: minimal + notes: no public real-time data; monthly statistics from the China Electricity Council and NEA + source_ids: [source:wikidata:sparql] + demand_range_mw: null + sources: [source:wikidata:sparql] + notes: "World's largest utility, covering ~88% of China's territory; not one synchronous system: North, Northeast, East, Central, Northwest and Southwest regional grids are linked largely by UHV AC/DC; massive west-to-east HVDC corridors." + + - grid_id: grid:cn-csg + name: China Southern Power Grid + kind: regional_grid + parent_grid: null + countries: [cn] + frequency_hz: 50 + timezone: Asia/Shanghai + operators: + - { name: China Southern Power Grid, role: system_operator } + - { name: China Southern Power Grid, role: transmission_owner } + - { name: National Energy Administration, role: regulator } + market: + structure: Guangdong spot market most advanced in China; regional market integration under way + price_mechanism: provincial spot prices where operating; regulated tariffs elsewhere + dispatch_interval: null + interconnections: [] + live_data: + availability: minimal + notes: no public real-time data; periodic statistics via the China Electricity Council + source_ids: [source:wikidata:sparql] + demand_range_mw: null + sources: [source:wikidata:sparql] + notes: "Serves Guangdong, Guangxi, Yunnan, Guizhou and Hainan; supplies Hong Kong and Macau via ties; Yunnan operates asynchronously (back-to-back HVDC) from the rest since 2016; extensive west-east HVDC." + + - grid_id: grid:in + name: Indian National Grid + kind: national_grid + parent_grid: null + countries: [in] + frequency_hz: 50 + timezone: Asia/Kolkata + operators: + - { name: Grid Controller of India (Grid-India), role: system_operator } + - { name: Power Grid Corporation of India, role: transmission_owner } + - { name: Indian Energy Exchange, role: market_operator } + - { name: Central Electricity Regulatory Commission, role: regulator } + market: + structure: long-term PPAs dominant; ~10% traded on exchanges (IEX, PXIL, HPX); real-time market since 2020 + price_mechanism: exchange market-clearing prices in 15-minute blocks + dispatch_interval: 15min + interconnections: [] + live_data: + availability: partial + notes: real-time national demand and frequency dashboards from Grid-India; daily power supply position reports; no unit-level API + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 120000, max: 250000 } + sources: [source:wikidata:sparql] + notes: "One-nation-one-grid: five regional grids fully synchronous since Dec 2013; synchronous ties to Bhutan and Nepal; HVDC back-to-back tie to Bangladesh; Sri Lanka link proposed." + + - grid_id: grid:kr + name: South Korea National Grid + kind: national_grid + parent_grid: null + countries: [kr] + frequency_hz: 60 + timezone: Asia/Seoul + operators: + - { name: Korea Power Exchange (KPX), role: system_operator } + - { name: Korea Power Exchange (KPX), role: market_operator } + - { name: Korea Electric Power Corporation (KEPCO), role: transmission_owner } + market: + structure: mandatory cost-based pool (CBP); KEPCO sole retailer + price_mechanism: system marginal price with fuel-cost pass-through + dispatch_interval: null + interconnections: [] + live_data: + availability: partial + notes: real-time supply-demand dashboard and EPSIS statistics from KPX; no dispatch-level API + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 45000, max: 100000 } + sources: [source:wikidata:sparql] + notes: "Electrically isolated national system (no international ties); Jeju island supplied via HVDC; nuclear and coal heavy with growing offshore wind plans." + + - grid_id: grid:tw + name: Taiwan Power Grid + kind: isolated_grid + parent_grid: null + countries: [tw] + frequency_hz: 60 + timezone: Asia/Taipei + operators: + - { name: Taiwan Power Company (Taipower), role: system_operator } + - { name: Taiwan Power Company (Taipower), role: transmission_owner } + - { name: Energy Administration MOEA, role: regulator } + market: + structure: vertically integrated single buyer; ancillary services trading platform since 2021 + price_mechanism: regulated tariffs; IPP contracts + dispatch_interval: null + interconnections: [] + live_data: + availability: partial + notes: 10-minute system generation mix and reserve margin published on Taipower open data + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 22000, max: 42000 } + sources: [source:wikidata:sparql] + notes: "Isolated 60 Hz island system; nuclear phase-out completed 2025; LNG-dominated with tight reserve margins; offshore wind buildout." + + - grid_id: grid:th + name: Thailand National Grid (EGAT) + kind: national_grid + parent_grid: null + countries: [th] + frequency_hz: 50 + timezone: Asia/Bangkok + operators: + - { name: Electricity Generating Authority of Thailand (EGAT), role: system_operator } + - { name: Electricity Generating Authority of Thailand (EGAT), role: transmission_owner } + - { name: Energy Regulatory Commission, role: regulator } + market: + structure: enhanced single buyer; EGAT purchases from IPPs and imports + price_mechanism: regulated tariffs with fuel adjustment (Ft) + dispatch_interval: null + interconnections: + - { to_grid: grid:my-peninsular, name: "HVDC link plus AC ties (Thailand-Malaysia)", capacity_mw: 300 } + live_data: + availability: partial + notes: real-time national demand dashboard from EGAT; periodic statistics from EPPO + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 18000, max: 36000 } + sources: [source:wikidata:sparql] + notes: "Major importer of Lao hydropower over radial AC and HVDC ties; transit leg of the Laos-Thailand-Malaysia-Singapore (LTMS) multilateral trade; gas-dominated fleet." + + - grid_id: grid:my-peninsular + name: Peninsular Malaysia Grid + kind: regional_grid + parent_grid: null + countries: [my] + frequency_hz: 50 + timezone: Asia/Kuala_Lumpur + operators: + - { name: Grid System Operator (GSO), role: system_operator } + - { name: Tenaga Nasional Berhad, role: transmission_owner } + - { name: Single Buyer, role: market_operator } + - { name: Energy Commission (Suruhanjaya Tenaga), role: regulator } + market: + structure: single-buyer model with incentive-based regulation; no wholesale spot market + price_mechanism: regulated tariffs with imbalance cost pass-through + dispatch_interval: null + interconnections: + - { to_grid: grid:th, name: "HVDC link plus AC ties (Thailand-Malaysia)", capacity_mw: 300 } + live_data: + availability: partial + notes: GSO publishes system demand and generation data; not real-time dispatch level + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 11000, max: 20000 } + sources: [source:wikidata:sparql] + notes: "Peninsular grid only: Sabah and Sarawak run separate systems on Borneo; AC tie to Singapore (~450 MW) carries LTMS trade; cross-border sales to Singapore expanding." + + - grid_id: grid:id-jamali + name: Java-Madura-Bali System (Jamali) + kind: regional_grid + parent_grid: null + countries: [id] + frequency_hz: 50 + timezone: Asia/Jakarta + operators: + - { name: Perusahaan Listrik Negara (PLN), role: system_operator } + - { name: Perusahaan Listrik Negara (PLN), role: transmission_owner } + - { name: Ministry of Energy and Mineral Resources, role: regulator } + market: + structure: vertically integrated single buyer (PLN) with IPP contracts + price_mechanism: regulated tariffs + dispatch_interval: null + interconnections: [] + live_data: + availability: minimal + notes: no public real-time data; PLN and ministry periodic statistics only + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 17000, max: 31000 } + sources: [source:wikidata:sparql] + notes: "Indonesia's dominant system (~70% of national demand) on a 500 kV backbone; other islands run separate PLN grids; coal-heavy with large captive-industry growth elsewhere." + + - grid_id: grid:ph + name: Philippine Grid (Luzon-Visayas-Mindanao) + kind: national_grid + parent_grid: null + countries: [ph] + frequency_hz: 60 + timezone: Asia/Manila + operators: + - { name: National Grid Corporation of the Philippines (NGCP), role: system_operator } + - { name: National Grid Corporation of the Philippines (NGCP), role: transmission_owner } + - { name: Independent Electricity Market Operator of the Philippines (IEMOP), role: market_operator } + - { name: Energy Regulatory Commission, role: regulator } + market: + structure: gross pool (WESM) spanning all three island grids + price_mechanism: nodal ex-ante pricing, 5-minute dispatch + dispatch_interval: 5min + interconnections: [] + live_data: + availability: partial + notes: WESM prices and dispatch data published by IEMOP as downloads rather than a real-time API; NGCP system bulletins + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 9000, max: 17000 } + sources: [source:wikidata:sparql] + notes: "Luzon and Visayas joined by 350 MW HVDC; Mindanao joined via the 450 MW Mindanao-Visayas Interconnection (energized 2023, full commercial operation Jan 2024), unifying the three grids and the WESM." + + - grid_id: grid:vn + name: Vietnam National Grid + kind: national_grid + parent_grid: null + countries: [vn] + frequency_hz: 50 + timezone: Asia/Ho_Chi_Minh + operators: + - { name: National Power System and Market Operator (NSMO), role: system_operator } + - { name: National Power System and Market Operator (NSMO), role: market_operator } + - { name: EVN National Power Transmission Corporation (EVNNPT), role: transmission_owner } + - { name: Electricity Regulatory Authority of Vietnam, role: regulator } + market: + structure: wholesale market (VWEM) in transition; EVN dominant buyer; DPPA scheme since 2024 + price_mechanism: cost-based dispatch with market price caps + dispatch_interval: null + interconnections: [] + live_data: + availability: minimal + notes: limited daily operational data from NSMO and EVN; no real-time public feed + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 25000, max: 51000 } + sources: [source:wikidata:sparql] + notes: "Long 500 kV north-south backbone; NSMO separated from EVN in Aug 2024 under the Ministry of Industry and Trade; imports from China and Laos over radial ties; world-scale rooftop solar boom of 2020." diff --git a/intelligence/data/registries/grids/au.yaml b/intelligence/data/registries/grids/au.yaml new file mode 100644 index 0000000..d0e00ae --- /dev/null +++ b/intelligence/data/registries/grids/au.yaml @@ -0,0 +1,229 @@ +# Grid registry — Australia. Class-A structural records. See ../../schemas/grid.schema.json +records: + - grid_id: grid:au-nem + name: National Electricity Market + kind: iso_rto + parent_grid: null + countries: [au] + frequency_hz: 50 + timezone: Australia/Brisbane + operators: + - { name: Australian Energy Market Operator, role: market_operator } + - { name: Australian Energy Market Operator, role: system_operator } + - { name: Australian Energy Regulator, role: regulator } + market: + structure: gross pool, energy-only + price_mechanism: regional reference price, 5-minute settlement + dispatch_interval: 5min + interconnections: [] + live_data: + availability: rich + notes: 5-minute public dispatch, price, demand and interconnector flow via NEMWEB + source_ids: [source:aemo:nemweb-dispatchis, source:aemo:nemweb-dispatch-scada, source:aemo:vis-nem-summary, source:openelectricity:api] + demand_range_mw: { min: 15000, max: 35000 } + sources: [source:aemo:vis-nem-summary, source:aemo:nem-registration-and-exemption-list] + notes: "Five regions (QLD1, NSW1, VIC1, SA1, TAS1); mainland is one of the world's longest synchronous systems; Tasmania is a synchronous island joined via Basslink HVDC; electrically isolated from the WEM/SWIS and NWIS." + + - grid_id: grid:au-nem-qld1 + name: NEM Queensland region (QLD1) + kind: regional_grid + parent_grid: grid:au-nem + countries: [au] + frequency_hz: 50 + timezone: Australia/Brisbane + operators: + - { name: Powerlink Queensland, role: transmission_owner } + market: null + interconnections: + - { to_grid: grid:au-nem-nsw1, name: "Queensland-NSW Interconnector (QNI)", capacity_mw: 1200 } + - { to_grid: grid:au-nem-nsw1, name: "Terranora (Directlink) HVDC", capacity_mw: 210 } + live_data: + availability: rich + notes: per-region 5-minute dispatch, price and demand via NEMWEB + source_ids: [source:aemo:nemweb-dispatchis, source:openelectricity:api] + demand_range_mw: { min: 4500, max: 11000 } + sources: [source:aemo:vis-nem-summary] + notes: "Coal-heavy but fast-growing rooftop solar; CopperString will extend this region to Mount Isa in the 2030s." + + - grid_id: grid:au-nem-nsw1 + name: NEM New South Wales region (NSW1) + kind: regional_grid + parent_grid: grid:au-nem + countries: [au] + frequency_hz: 50 + timezone: Australia/Sydney + operators: + - { name: Transgrid, role: transmission_owner } + market: null + interconnections: + - { to_grid: grid:au-nem-qld1, name: "Queensland-NSW Interconnector (QNI)", capacity_mw: 1200 } + - { to_grid: grid:au-nem-vic1, name: "Victoria-NSW Interconnector (VNI)", capacity_mw: 1600 } + - { to_grid: grid:au-nem-sa1, name: "Project EnergyConnect stage 1", capacity_mw: 150 } + live_data: + availability: rich + notes: per-region 5-minute dispatch, price and demand via NEMWEB + source_ids: [source:aemo:nemweb-dispatchis, source:openelectricity:api] + demand_range_mw: { min: 5500, max: 15000 } + sources: [source:aemo:vis-nem-summary] + notes: "Largest NEM region by demand; Snowy Hydro pumped storage; Project EnergyConnect to reach 800 MW when complete." + + - grid_id: grid:au-nem-vic1 + name: NEM Victoria region (VIC1) + kind: regional_grid + parent_grid: grid:au-nem + countries: [au] + frequency_hz: 50 + timezone: Australia/Melbourne + operators: + - { name: AusNet Services, role: transmission_owner } + market: null + interconnections: + - { to_grid: grid:au-nem-nsw1, name: "Victoria-NSW Interconnector (VNI)", capacity_mw: 1600 } + - { to_grid: grid:au-nem-sa1, name: "Heywood Interconnector", capacity_mw: 650 } + - { to_grid: grid:au-nem-sa1, name: "Murraylink HVDC", capacity_mw: 220 } + - { to_grid: grid:au-nem-tas1, name: "Basslink HVDC", capacity_mw: 500 } + live_data: + availability: rich + notes: per-region 5-minute dispatch, price and demand via NEMWEB + source_ids: [source:aemo:nemweb-dispatchis, source:openelectricity:api] + demand_range_mw: { min: 3800, max: 10500 } + sources: [source:aemo:vis-nem-summary] + notes: "Brown-coal Latrobe Valley fleet retiring; hub for Basslink and planned Marinus Link to Tasmania." + + - grid_id: grid:au-nem-sa1 + name: NEM South Australia region (SA1) + kind: regional_grid + parent_grid: grid:au-nem + countries: [au] + frequency_hz: 50 + timezone: Australia/Adelaide + operators: + - { name: ElectraNet, role: transmission_owner } + market: null + interconnections: + - { to_grid: grid:au-nem-vic1, name: "Heywood Interconnector", capacity_mw: 650 } + - { to_grid: grid:au-nem-vic1, name: "Murraylink HVDC", capacity_mw: 220 } + - { to_grid: grid:au-nem-nsw1, name: "Project EnergyConnect stage 1", capacity_mw: 150 } + live_data: + availability: rich + notes: per-region 5-minute dispatch, price and demand via NEMWEB + source_ids: [source:aemo:nemweb-dispatchis, source:openelectricity:api] + demand_range_mw: { min: 900, max: 3300 } + sources: [source:aemo:vis-nem-summary] + notes: "World-leading wind and solar share; regularly runs at very high instantaneous renewables; grid-forming batteries and synchronous condensers for strength." + + - grid_id: grid:au-nem-tas1 + name: NEM Tasmania region (TAS1) + kind: regional_grid + parent_grid: grid:au-nem + countries: [au] + frequency_hz: 50 + timezone: Australia/Hobart + operators: + - { name: TasNetworks, role: transmission_owner } + market: null + interconnections: + - { to_grid: grid:au-nem-vic1, name: "Basslink HVDC", capacity_mw: 500 } + live_data: + availability: rich + notes: per-region 5-minute dispatch, price and demand via NEMWEB + source_ids: [source:aemo:nemweb-dispatchis, source:openelectricity:api] + demand_range_mw: { min: 900, max: 1900 } + sources: [source:aemo:vis-nem-summary] + notes: "Hydro-dominated synchronous island within the NEM, joined to Victoria only by Basslink HVDC; Marinus Link (750 MW stage 1) planned." + + - grid_id: grid:au-wem + name: South West Interconnected System (WEM) + kind: isolated_grid + parent_grid: null + countries: [au] + frequency_hz: 50 + timezone: Australia/Perth + operators: + - { name: Australian Energy Market Operator, role: market_operator } + - { name: Australian Energy Market Operator, role: system_operator } + - { name: Western Power, role: transmission_owner } + - { name: Economic Regulation Authority of WA, role: regulator } + market: + structure: energy market plus reserve capacity mechanism; security-constrained economic dispatch since the 2023 reforms + price_mechanism: marginal energy pricing with capacity credits + dispatch_interval: 5min + interconnections: [] + live_data: + availability: rich + notes: 5-minute facility SCADA, prices, load and outage data published on the public WEM data portal + source_ids: [source:aemo:wem-data-portal, source:openelectricity:api] + demand_range_mw: { min: 1100, max: 4300 } + sources: [source:aemo:wem-data-portal] + notes: "The Wholesale Electricity Market runs on the SWIS serving Perth and south-west WA; electrically isolated from every other grid (nearest, the NEM, is ~2000 km away)." + + - grid_id: grid:au-nwis + name: North West Interconnected System + kind: isolated_grid + parent_grid: null + countries: [au] + frequency_hz: 50 + timezone: Australia/Perth + operators: + - { name: Pilbara ISOCo, role: system_operator } + - { name: Horizon Power, role: transmission_owner } + - { name: Alinta Energy, role: transmission_owner } + - { name: Rio Tinto, role: transmission_owner } + market: + structure: bilateral contracts under the light-handed Pilbara networks access regime; no pool + price_mechanism: null + dispatch_interval: null + interconnections: [] + live_data: + availability: minimal + notes: no public operational feed; periodic Pilbara ISOCo and WA regulator reports only + source_ids: [source:wikidata:sparql] + demand_range_mw: null + sources: [source:wikidata:sparql] + notes: "Pilbara mining grid of interconnected private and state-owned networks (Horizon, Alinta, Rio Tinto, plus BHP/FMG assets nearby); Pilbara ISOCo is the appointed ISO with real-time coordination performed by Horizon Power; isolated from the SWIS." + + - grid_id: grid:au-dkis + name: Darwin-Katherine Interconnected System + kind: isolated_grid + parent_grid: null + countries: [au] + frequency_hz: 50 + timezone: Australia/Darwin + operators: + - { name: Power and Water Corporation, role: system_operator } + - { name: Northern Territory Electricity System and Market Operator, role: market_operator } + - { name: Utilities Commission of the Northern Territory, role: regulator } + market: + structure: interim NT wholesale market administered by NTESMO; Territory Generation dominant + price_mechanism: daily published market prices + dispatch_interval: null + interconnections: [] + live_data: + availability: partial + notes: daily price, demand and trading publications from NTESMO; no real-time API + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 130, max: 300 } + sources: [source:wikidata:sparql] + notes: "Largest of the three regulated NT systems (others are Alice Springs and Tennant Creek); very high rooftop solar penetration; gas-fired backbone." + + - grid_id: grid:au-mtisa + name: North West Power System (Mount Isa) + kind: isolated_grid + parent_grid: null + countries: [au] + frequency_hz: 50 + timezone: Australia/Brisbane + operators: + - { name: Ergon Energy (Energy Queensland), role: transmission_owner } + market: + structure: bilateral contracts with mining loads; no organized market + price_mechanism: null + dispatch_interval: null + interconnections: [] + live_data: + availability: none + notes: no public operational data; the real portal is Queensland government and AER periodic reporting + source_ids: [] + demand_range_mw: null + sources: [source:wikidata:sparql] + notes: "Supplies the Mount Isa-Cloncurry mining district; generation dominated by APA Group's Diamantina gas-fired power station; CopperString transmission project (under construction, target early 2030s) will connect it to the NEM." diff --git a/intelligence/data/registries/grids/europe.yaml b/intelligence/data/registries/grids/europe.yaml new file mode 100644 index 0000000..393f682 --- /dev/null +++ b/intelligence/data/registries/grids/europe.yaml @@ -0,0 +1,493 @@ +# Grid registry — Europe. Class-A structural records. See ../../schemas/grid.schema.json +records: + - grid_id: grid:eu-continental + name: Continental Europe Synchronous Area + kind: synchronous_area + parent_grid: null + countries: [de, fr, es, pt, it, pl, nl, be, lu, ch, at, cz, sk, hu, si, hr, ro, bg, gr, rs, ba, me, mk, al, dk, ua, md, tr, ee, lv, lt] + frequency_hz: 50 + timezone: Europe/Brussels + operators: + - { name: "ENTSO-E Regional Group Continental Europe (coordination of member TSOs)", role: system_operator } + market: + structure: EU internal electricity market; zonal bidding areas coupled via SDAC/SIDC + price_mechanism: zonal marginal pricing in coupled day-ahead and intraday auctions + dispatch_interval: null + interconnections: + - { to_grid: grid:eu-nordic, name: "multiple HVDC links (Skagerrak, Kontek, Baltic Cable, NordLink, NorNed, COBRA and others)", capacity_mw: null } + - { to_grid: grid:gb, name: "HVDC links from France, Belgium, Netherlands and Denmark", capacity_mw: null } + live_data: + availability: rich + notes: pan-area load, generation and cross-border flows via the ENTSO-E transparency platform + source_ids: [source:entsoe:transparency-platform] + demand_range_mw: null + sources: [source:entsoe:transparency-platform] + notes: "World's largest synchronous area by connected capacity; includes west Denmark (DK1) only; Ukraine/Moldova joined Mar 2022, Baltic states Feb 2025, Turkey synchronized since 2010." + + - grid_id: grid:eu-nordic + name: Nordic Synchronous Area + kind: synchronous_area + parent_grid: null + countries: ["no", se, fi, dk] + frequency_hz: 50 + timezone: Europe/Oslo + operators: + - { name: Statnett, role: tso } + - { name: Svenska kraftnat, role: tso } + - { name: Fingrid, role: tso } + - { name: Energinet, role: tso } + market: + structure: Nord Pool day-ahead and intraday markets across zonal bidding areas + price_mechanism: zonal marginal pricing; 15-minute imbalance settlement + dispatch_interval: null + interconnections: + - { to_grid: grid:eu-continental, name: "multiple HVDC links to Germany, Netherlands, Poland and DK1", capacity_mw: null } + - { to_grid: grid:gb, name: "North Sea Link (Norway-GB HVDC)", capacity_mw: 1400 } + - { to_grid: grid:eu-baltic, name: "EstLink 1+2 (Finland-Estonia) and NordBalt (Sweden-Lithuania) HVDC", capacity_mw: 1700 } + live_data: + availability: rich + notes: TSO and market data via ENTSO-E transparency and Nord Pool + source_ids: [source:entsoe:transparency-platform, source:nordpool:market-data] + demand_range_mw: null + sources: [source:entsoe:transparency-platform, source:nordpool:market-data] + notes: "Hydro-dominated synchronous system of Norway, Sweden, Finland and east Denmark (DK2); joint Nordic balancing model." + + - grid_id: grid:eu-baltic + name: Baltic States Grid (Estonia-Latvia-Lithuania) + kind: regional_grid + parent_grid: grid:eu-continental + countries: [ee, lv, lt] + frequency_hz: 50 + timezone: Europe/Vilnius + operators: + - { name: Elering, role: tso } + - { name: Augstsprieguma tikls (AST), role: tso } + - { name: Litgrid, role: tso } + market: + structure: Nord Pool bidding zones (EE, LV, LT) within the EU coupled market + price_mechanism: zonal marginal pricing via SDAC/SIDC + dispatch_interval: null + interconnections: + - { to_grid: grid:pl, name: "LitPol Link (AC, the synchronous tie to Continental Europe)", capacity_mw: 500 } + - { to_grid: grid:fi, name: "EstLink 1+2 HVDC", capacity_mw: 1000 } + - { to_grid: grid:se, name: "NordBalt HVDC", capacity_mw: 700 } + live_data: + availability: rich + notes: full ENTSO-E transparency coverage plus Nord Pool market data + source_ids: [source:entsoe:transparency-platform, source:nordpool:market-data] + demand_range_mw: null + sources: [source:entsoe:transparency-platform] + notes: "Desynchronized from the Russian-controlled BRELL ring on 8 Feb 2025 and synchronized with Continental Europe on 9 Feb 2025 via LitPol; Harmony Link HVDC to Poland planned." + + - grid_id: grid:gb + name: Great Britain Synchronous Area + kind: synchronous_area + parent_grid: null + countries: [gb] + frequency_hz: 50 + timezone: Europe/London + operators: + - { name: National Energy System Operator (NESO), role: system_operator } + - { name: National Grid Electricity Transmission, role: transmission_owner } + - { name: Elexon, role: market_operator } + - { name: Ofgem, role: regulator } + market: + structure: bilateral trading with a balancing mechanism (BETTA) + price_mechanism: pay-as-bid balancing mechanism; half-hourly imbalance settlement + dispatch_interval: 30min + interconnections: + - { to_grid: grid:fr, name: "IFA, IFA2 and ElecLink HVDC", capacity_mw: 4000 } + - { to_grid: grid:be, name: "Nemo Link HVDC", capacity_mw: 1000 } + - { to_grid: grid:nl, name: "BritNed HVDC", capacity_mw: 1000 } + - { to_grid: grid:no, name: "North Sea Link HVDC", capacity_mw: 1400 } + - { to_grid: grid:dk, name: "Viking Link HVDC", capacity_mw: 1400 } + - { to_grid: grid:ie-sem, name: "Moyle, EWIC and Greenlink HVDC", capacity_mw: 1500 } + live_data: + availability: rich + notes: unit-level balancing mechanism, generation, demand and price data via Elexon BMRS/Insights API + source_ids: [source:elexon:bmrs] + demand_range_mw: { min: 15000, max: 48000 } + sources: [source:elexon:bmrs] + notes: "Own synchronous island; NESO (publicly owned, replaced National Grid ESO in Oct 2024); Scottish TOs are SSEN and SP Energy Networks; Northern Ireland belongs to the Ireland synchronous area, not this one." + + - grid_id: grid:ie-sem + name: Single Electricity Market (island of Ireland) + kind: synchronous_area + parent_grid: null + countries: [ie, gb] + frequency_hz: 50 + timezone: Europe/Dublin + operators: + - { name: EirGrid, role: tso } + - { name: SONI, role: tso } + - { name: SEMO, role: market_operator } + market: + structure: all-island gross pool (I-SEM) with EU day-ahead/intraday coupling and a balancing market + price_mechanism: marginal pricing; half-hourly imbalance settlement + dispatch_interval: 5min + interconnections: + - { to_grid: grid:gb, name: "Moyle, EWIC and Greenlink HVDC", capacity_mw: 1500 } + live_data: + availability: rich + notes: real-time system dashboards from EirGrid/SONI, SEMO market data, ENTSO-E transparency coverage + source_ids: [source:entsoe:transparency-platform] + demand_range_mw: { min: 2500, max: 7500 } + sources: [source:entsoe:transparency-platform] + notes: "Single synchronous area and market spanning Ireland and Northern Ireland; world-leading system non-synchronous penetration limits (75%); Celtic Interconnector to France (700 MW) under construction." + + - grid_id: grid:de + name: Germany (DE-LU bidding zone) + kind: tso_area + parent_grid: grid:eu-continental + countries: [de] + frequency_hz: 50 + timezone: Europe/Berlin + operators: + - { name: 50Hertz Transmission, role: tso } + - { name: Amprion, role: tso } + - { name: TenneT TSO GmbH, role: tso } + - { name: TransnetBW, role: tso } + - { name: Bundesnetzagentur, role: regulator } + market: + structure: single DE-LU bidding zone across four TSO areas; self-dispatch + price_mechanism: zonal marginal pricing via SDAC/SIDC; quarter-hourly intraday products + dispatch_interval: null + interconnections: [] + live_data: + availability: rich + notes: generation, load, prices and flows via ENTSO-E transparency and the national SMARD portal + source_ids: [source:entsoe:transparency-platform] + demand_range_mw: { min: 35000, max: 80000 } + sources: [source:entsoe:transparency-platform] + notes: "Europe's largest power system; four TSOs share one bidding zone (zone-split debate ongoing); heavy north-south congestion and redispatch." + + - grid_id: grid:fr + name: France (RTE area) + kind: tso_area + parent_grid: grid:eu-continental + countries: [fr] + frequency_hz: 50 + timezone: Europe/Paris + operators: + - { name: RTE (Reseau de Transport d'Electricite), role: tso } + - { name: Commission de Regulation de l'Energie, role: regulator } + market: + structure: single bidding zone; self-dispatch; EPEX Spot trading + price_mechanism: zonal marginal pricing via SDAC/SIDC + dispatch_interval: null + interconnections: [] + live_data: + availability: rich + notes: ENTSO-E transparency plus RTE's open eco2mix real-time data + source_ids: [source:entsoe:transparency-platform] + demand_range_mw: { min: 30000, max: 102000 } + sources: [source:entsoe:transparency-platform] + notes: "Nuclear-dominated (~65% of output); Europe's most temperature-sensitive demand; major exporter in normal years." + + - grid_id: grid:es + name: Spain (Red Electrica area) + kind: tso_area + parent_grid: grid:eu-continental + countries: [es] + frequency_hz: 50 + timezone: Europe/Madrid + operators: + - { name: Red Electrica de Espana, role: tso } + - { name: OMIE, role: market_operator } + - { name: CNMC, role: regulator } + market: + structure: MIBEL Iberian market with Portugal; self-dispatch + price_mechanism: zonal marginal pricing via SDAC/SIDC + dispatch_interval: null + interconnections: [] + live_data: + availability: rich + notes: ENTSO-E transparency plus REE's real-time demand and generation data + source_ids: [source:entsoe:transparency-platform] + demand_range_mw: { min: 20000, max: 45000 } + sources: [source:entsoe:transparency-platform] + notes: "Weakly interconnected with France (electrical peninsula); April 2025 Iberian blackout prompted grid-strength reforms; Balearics linked by HVDC, Canaries isolated." + + - grid_id: grid:it + name: Italy (Terna area) + kind: tso_area + parent_grid: grid:eu-continental + countries: [it] + frequency_hz: 50 + timezone: Europe/Rome + operators: + - { name: Terna, role: tso } + - { name: Gestore dei Mercati Energetici (GME), role: market_operator } + - { name: ARERA, role: regulator } + market: + structure: seven bidding zones; national single price for consumers (PUN) + price_mechanism: zonal marginal pricing via SDAC/SIDC + dispatch_interval: null + interconnections: [] + live_data: + availability: rich + notes: ENTSO-E transparency plus Terna's public real-time system data + source_ids: [source:entsoe:transparency-platform] + demand_range_mw: { min: 20000, max: 60000 } + sources: [source:entsoe:transparency-platform] + notes: "Zonal north-south structure; HVDC links to Sardinia, Montenegro and Greece; gas-heavy generation mix." + + - grid_id: grid:pl + name: Poland (PSE area) + kind: tso_area + parent_grid: grid:eu-continental + countries: [pl] + frequency_hz: 50 + timezone: Europe/Warsaw + operators: + - { name: Polskie Sieci Elektroenergetyczne (PSE), role: tso } + - { name: Towarowa Gielda Energii (TGE), role: market_operator } + - { name: Urzad Regulacji Energetyki, role: regulator } + market: + structure: single bidding zone; central dispatch of large units + price_mechanism: zonal marginal pricing via SDAC/SIDC plus capacity market + dispatch_interval: null + interconnections: [] + live_data: + availability: rich + notes: ENTSO-E transparency plus PSE's public operational data + source_ids: [source:entsoe:transparency-platform] + demand_range_mw: { min: 11000, max: 28000 } + sources: [source:entsoe:transparency-platform] + notes: "Coal-heavy but rapidly adding solar; hosts LitPol, the Baltic states' synchronous gateway to Continental Europe." + + - grid_id: grid:nl + name: Netherlands (TenneT area) + kind: tso_area + parent_grid: grid:eu-continental + countries: [nl] + frequency_hz: 50 + timezone: Europe/Amsterdam + operators: + - { name: TenneT TSO B.V., role: tso } + - { name: Autoriteit Consument en Markt, role: regulator } + market: + structure: single bidding zone; self-dispatch + price_mechanism: zonal marginal pricing via SDAC/SIDC + dispatch_interval: null + interconnections: [] + live_data: + availability: rich + notes: ENTSO-E transparency plus TenneT operational data + source_ids: [source:entsoe:transparency-platform] + demand_range_mw: { min: 8000, max: 19000 } + sources: [source:entsoe:transparency-platform] + notes: "Large offshore wind buildout; severe distribution-level grid congestion (netcongestie) constraining new connections." + + - grid_id: grid:be + name: Belgium (Elia area) + kind: tso_area + parent_grid: grid:eu-continental + countries: [be] + frequency_hz: 50 + timezone: Europe/Brussels + operators: + - { name: Elia, role: tso } + - { name: CREG, role: regulator } + market: + structure: single bidding zone; self-dispatch + price_mechanism: zonal marginal pricing via SDAC/SIDC + dispatch_interval: null + interconnections: [] + live_data: + availability: rich + notes: ENTSO-E transparency plus Elia's open data portal + source_ids: [source:entsoe:transparency-platform] + demand_range_mw: { min: 6000, max: 14000 } + sources: [source:entsoe:transparency-platform] + notes: "Nuclear life extensions (Doel 4, Tihange 3) reversed the phase-out; Nemo Link to GB and Nautilus planned." + + - grid_id: grid:ch + name: Switzerland (Swissgrid area) + kind: tso_area + parent_grid: grid:eu-continental + countries: [ch] + frequency_hz: 50 + timezone: Europe/Zurich + operators: + - { name: Swissgrid, role: tso } + - { name: ElCom, role: regulator } + market: + structure: national market outside EU market coupling (no EU electricity agreement in force) + price_mechanism: exchange trading with explicit border capacity allocation + dispatch_interval: null + interconnections: [] + live_data: + availability: rich + notes: ENTSO-E transparency coverage; Swissgrid operational data + source_ids: [source:entsoe:transparency-platform] + demand_range_mw: { min: 4000, max: 10000 } + sources: [source:entsoe:transparency-platform] + notes: "Alpine hydro storage hub at the heart of the Continental grid; heavily transited despite exclusion from EU coupling." + + - grid_id: grid:at + name: Austria (APG area) + kind: tso_area + parent_grid: grid:eu-continental + countries: [at] + frequency_hz: 50 + timezone: Europe/Vienna + operators: + - { name: Austrian Power Grid, role: tso } + - { name: E-Control, role: regulator } + market: + structure: single bidding zone (split from DE-AT zone in 2018); self-dispatch + price_mechanism: zonal marginal pricing via SDAC/SIDC + dispatch_interval: null + interconnections: [] + live_data: + availability: rich + notes: ENTSO-E transparency plus APG operational data + source_ids: [source:entsoe:transparency-platform] + demand_range_mw: { min: 5000, max: 12000 } + sources: [source:entsoe:transparency-platform] + notes: "Hydro-dominated; pumped-storage services for Germany; Vorarlberg area is electrically closer to the German grid." + + - grid_id: grid:no + name: Norway (Statnett area) + kind: tso_area + parent_grid: grid:eu-nordic + countries: ["no"] + frequency_hz: 50 + timezone: Europe/Oslo + operators: + - { name: Statnett, role: tso } + - { name: NVE-RME, role: regulator } + market: + structure: five bidding zones (NO1-NO5) in Nord Pool + price_mechanism: zonal marginal pricing + dispatch_interval: null + interconnections: [] + live_data: + availability: rich + notes: ENTSO-E transparency plus Nord Pool and Statnett data + source_ids: [source:entsoe:transparency-platform, source:nordpool:market-data] + demand_range_mw: { min: 10000, max: 25000 } + sources: [source:entsoe:transparency-platform, source:nordpool:market-data] + notes: "~90% hydro; Europe's battery via NordLink, NorNed and North Sea Link HVDC exports; strong internal north-south price splits." + + - grid_id: grid:se + name: Sweden (Svenska kraftnat area) + kind: tso_area + parent_grid: grid:eu-nordic + countries: [se] + frequency_hz: 50 + timezone: Europe/Stockholm + operators: + - { name: Svenska kraftnat, role: tso } + - { name: Energimarknadsinspektionen, role: regulator } + market: + structure: four bidding zones (SE1-SE4) in Nord Pool + price_mechanism: zonal marginal pricing + dispatch_interval: null + interconnections: [] + live_data: + availability: rich + notes: ENTSO-E transparency plus Nord Pool and SvK data + source_ids: [source:entsoe:transparency-platform, source:nordpool:market-data] + demand_range_mw: { min: 8000, max: 27000 } + sources: [source:entsoe:transparency-platform, source:nordpool:market-data] + notes: "Hydro north, nuclear south, chronic SE2/SE3 transfer constraint; NordBalt tie to Lithuania." + + - grid_id: grid:fi + name: Finland (Fingrid area) + kind: tso_area + parent_grid: grid:eu-nordic + countries: [fi] + frequency_hz: 50 + timezone: Europe/Helsinki + operators: + - { name: Fingrid, role: tso } + - { name: Energiavirasto, role: regulator } + market: + structure: single bidding zone in Nord Pool + price_mechanism: zonal marginal pricing + dispatch_interval: null + interconnections: [] + live_data: + availability: rich + notes: ENTSO-E transparency plus Fingrid's open data API + source_ids: [source:entsoe:transparency-platform, source:nordpool:market-data] + demand_range_mw: { min: 5000, max: 15000 } + sources: [source:entsoe:transparency-platform, source:nordpool:market-data] + notes: "Olkiluoto 3 made Finland near self-sufficient; Russian imports halted 2022; EstLink HVDC ties to Estonia." + + - grid_id: grid:dk + name: Denmark (Energinet area) + kind: tso_area + parent_grid: null + countries: [dk] + frequency_hz: 50 + timezone: Europe/Copenhagen + operators: + - { name: Energinet, role: tso } + - { name: Forsyningstilsynet, role: regulator } + market: + structure: two bidding zones in different synchronous areas; Nord Pool trading + price_mechanism: zonal marginal pricing + dispatch_interval: null + interconnections: + - { to_grid: grid:no, name: "Skagerrak HVDC", capacity_mw: 1700 } + - { to_grid: grid:se, name: "Konti-Skan HVDC plus Oresund AC", capacity_mw: 2400 } + - { to_grid: grid:de, name: "Jutland AC ties plus Kontek HVDC", capacity_mw: 3500 } + - { to_grid: grid:nl, name: "COBRAcable HVDC", capacity_mw: 700 } + - { to_grid: grid:gb, name: "Viking Link HVDC", capacity_mw: 1400 } + live_data: + availability: rich + notes: ENTSO-E transparency plus Energinet's open Energi Data Service API + source_ids: [source:entsoe:transparency-platform, source:nordpool:market-data] + demand_range_mw: { min: 2000, max: 6500 } + sources: [source:entsoe:transparency-platform] + notes: "Split system: DK1 (Jutland/Funen) is in the Continental Europe synchronous area, DK2 (Zealand) in the Nordic area, joined internally by the 600 MW Great Belt HVDC; parent_grid left null because of the split." + + - grid_id: grid:ua + name: Ukraine (Ukrenergo area) + kind: tso_area + parent_grid: grid:eu-continental + countries: [ua] + frequency_hz: 50 + timezone: Europe/Kyiv + operators: + - { name: NPC Ukrenergo, role: tso } + - { name: NEURC, role: regulator } + market: + structure: day-ahead, intraday and balancing markets since 2019; price caps in force + price_mechanism: zonal marginal pricing with administrative caps + dispatch_interval: null + interconnections: [] + live_data: + availability: partial + notes: partial ENTSO-E transparency coverage; wartime restrictions limit published operational data + source_ids: [source:entsoe:transparency-platform] + demand_range_mw: { min: 9000, max: 18000 } + sources: [source:entsoe:transparency-platform] + notes: "Emergency-synchronized with Continental Europe on 16 Mar 2022 (together with Moldova), hours after disconnecting from the Russian IPS/UPS; commercial exchanges with EU neighbours since Jun 2022; infrastructure under repeated wartime attack." + + - grid_id: grid:tr + name: Turkey (TEIAS area) + kind: tso_area + parent_grid: grid:eu-continental + countries: [tr] + frequency_hz: 50 + timezone: Europe/Istanbul + operators: + - { name: TEIAS, role: tso } + - { name: EPIAS (EXIST), role: market_operator } + - { name: EMRA, role: regulator } + market: + structure: day-ahead, intraday and balancing power markets operated by EPIAS + price_mechanism: hourly market clearing price (moving to quarter-hourly) + dispatch_interval: null + interconnections: [] + live_data: + availability: partial + notes: hourly generation, consumption and price data via the EPIAS transparency platform (free registration); limited ENTSO-E coverage + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 25000, max: 56000 } + sources: [source:wikidata:sparql] + notes: "Synchronous with Continental Europe via Bulgaria and Greece since the 2010 trial, permanent under the 2015 long-term agreement; ENTSO-E observer member; also asynchronous ties toward Georgia, Iraq and Iran." diff --git a/intelligence/data/registries/grids/isolated.yaml b/intelligence/data/registries/grids/isolated.yaml new file mode 100644 index 0000000..b087098 --- /dev/null +++ b/intelligence/data/registries/grids/isolated.yaml @@ -0,0 +1,161 @@ +# Grid registry — isolated & notable island systems. Class-A structural records. See ../../schemas/grid.schema.json +records: + - grid_id: grid:is + name: Iceland National Grid + kind: isolated_grid + parent_grid: null + countries: [is] + frequency_hz: 50 + timezone: Atlantic/Reykjavik + operators: + - { name: Landsnet, role: system_operator } + - { name: Landsnet, role: transmission_owner } + - { name: Orkustofnun, role: regulator } + market: + structure: bilateral contracts; energy-intensive industry takes ~80% of demand; nascent trading platform + price_mechanism: negotiated long-term contracts; regulated transmission tariffs + dispatch_interval: null + interconnections: [] + live_data: + availability: partial + notes: near-real-time system map and operational data published by Landsnet; no dispatch-level API + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 1600, max: 2400 } + sources: [source:wikidata:sparql] + notes: "Fully isolated island grid, ~100% renewable (hydro plus geothermal); very flat load dominated by aluminium smelters; IceLink interconnector proposals shelved." + + - grid_id: grid:nz + name: New Zealand Power System + kind: national_grid + parent_grid: null + countries: [nz] + frequency_hz: 50 + timezone: Pacific/Auckland + operators: + - { name: Transpower, role: system_operator } + - { name: Transpower, role: transmission_owner } + - { name: Electricity Authority, role: regulator } + market: + structure: energy-only gross pool (NZEM) with nodal pricing + price_mechanism: real-time 5-minute dispatch prices; 30-minute trading periods + dispatch_interval: 5min + interconnections: [] + live_data: + availability: rich + notes: 5-minute prices, load and generation freely available (WITS free data, em6 dashboards, Transpower live data) + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 4000, max: 7200 } + sources: [source:wikidata:sparql] + notes: "No international interconnections; two AC islands joined by the 1200 MW HVDC Inter-Island link; ~85% renewable, dominated by South Island hydro; dry-year risk is the defining security question." + + - grid_id: grid:nz-north + name: New Zealand North Island Grid + kind: regional_grid + parent_grid: grid:nz + countries: [nz] + frequency_hz: 50 + timezone: Pacific/Auckland + operators: + - { name: Transpower, role: system_operator } + market: null + interconnections: + - { to_grid: grid:nz-south, name: "HVDC Inter-Island link (Benmore-Haywards)", capacity_mw: 1200 } + live_data: + availability: rich + notes: covered by the same 5-minute national market data as the parent record + source_ids: [source:wikidata:sparql] + demand_range_mw: null + sources: [source:wikidata:sparql] + notes: "Auckland and Wellington load centres; geothermal, hydro and gas generation; net importer over the HVDC in dry southern years." + + - grid_id: grid:nz-south + name: New Zealand South Island Grid + kind: regional_grid + parent_grid: grid:nz + countries: [nz] + frequency_hz: 50 + timezone: Pacific/Auckland + operators: + - { name: Transpower, role: system_operator } + market: null + interconnections: + - { to_grid: grid:nz-north, name: "HVDC Inter-Island link (Benmore-Haywards)", capacity_mw: 1200 } + live_data: + availability: rich + notes: covered by the same 5-minute national market data as the parent record + source_ids: [source:wikidata:sparql] + demand_range_mw: null + sources: [source:wikidata:sparql] + notes: "Hydro-dominated (Benmore, Manapouri, Clyde); Tiwai Point aluminium smelter is the single largest load; usually exports north over the HVDC." + + - grid_id: grid:us-hi + name: Hawaii Island Grids + kind: isolated_grid + parent_grid: null + countries: [us] + frequency_hz: 60 + timezone: Pacific/Honolulu + operators: + - { name: Hawaiian Electric, role: system_operator } + - { name: Kauai Island Utility Cooperative, role: system_operator } + - { name: Hawaii Public Utilities Commission, role: regulator } + market: + structure: vertically integrated regulated utilities under performance-based regulation + price_mechanism: regulated tariffs; renewable PPAs + dispatch_interval: null + interconnections: [] + live_data: + availability: minimal + notes: no real-time public feed; EIA-930 excludes Hawaii; the real portals are monthly EIA data and utility dashboards + source_ids: [source:eia:api-v2] + demand_range_mw: { min: 700, max: 1800 } + sources: [source:eia:api-v2] + notes: "Umbrella record for six separate island systems (Oahu, Maui, Hawaii Island, Kauai, Molokai, Lanai) with no inter-island cables; among the world's highest rooftop-PV penetration; 100% renewable mandate by 2045." + + - grid_id: grid:pr + name: Puerto Rico Grid + kind: isolated_grid + parent_grid: null + countries: [us] + frequency_hz: 60 + timezone: America/Puerto_Rico + operators: + - { name: LUMA Energy, role: system_operator } + - { name: LUMA Energy, role: transmission_owner } + - { name: Puerto Rico Energy Bureau, role: regulator } + market: + structure: PREPA-owned assets under private operation (LUMA for T&D since 2021, Genera PR for generation since 2023) + price_mechanism: regulated tariffs + dispatch_interval: null + interconnections: [] + live_data: + availability: minimal + notes: LUMA outage and system dashboards only; EIA-930 excludes Puerto Rico; monthly EIA territory data + source_ids: [source:eia:api-v2] + demand_range_mw: { min: 1800, max: 3100 } + sources: [source:eia:api-v2] + notes: "Isolated 60 Hz island system still rebuilding from Hurricane Maria (2017); chronic generation shortfalls and island-wide outages; large federally funded solar-plus-storage program." + + - grid_id: grid:lk + name: Sri Lanka National Grid + kind: isolated_grid + parent_grid: null + countries: [lk] + frequency_hz: 50 + timezone: Asia/Colombo + operators: + - { name: Ceylon Electricity Board, role: system_operator } + - { name: Ceylon Electricity Board, role: transmission_owner } + - { name: Public Utilities Commission of Sri Lanka, role: regulator } + market: + structure: vertically integrated single buyer (CEB); unbundling legislated 2024 + price_mechanism: regulated tariffs + dispatch_interval: null + interconnections: [] + live_data: + availability: minimal + notes: daily generation reports and dashboards from CEB; no real-time public feed + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 1500, max: 2900 } + sources: [source:wikidata:sparql] + notes: "Isolated 50 Hz island grid; hydro plus imported-fuel thermal with growing solar; HVDC interconnection with India long proposed; 2022 fuel crisis forced rolling cuts." diff --git a/intelligence/data/registries/grids/north-america.yaml b/intelligence/data/registries/grids/north-america.yaml new file mode 100644 index 0000000..45af0c4 --- /dev/null +++ b/intelligence/data/registries/grids/north-america.yaml @@ -0,0 +1,333 @@ +# Grid registry — North America. Class-A structural records. See ../../schemas/grid.schema.json +records: + - grid_id: grid:us-eastern + name: Eastern Interconnection + kind: interconnection + parent_grid: null + countries: [us, ca] + frequency_hz: 60 + timezone: America/New_York + operators: + - { name: North American Electric Reliability Corporation, role: regulator } + market: null + interconnections: + - { to_grid: grid:us-western, name: "back-to-back HVDC ties", capacity_mw: 1300 } + - { to_grid: grid:us-texas, name: "DC ties (North and East)", capacity_mw: 820 } + - { to_grid: grid:ca-hydro-quebec, name: "HVDC and variable-frequency-transformer ties", capacity_mw: 5500 } + live_data: + availability: partial + notes: hourly balancing-authority-level demand, generation and interchange via EIA-930 + source_ids: [source:eia:eia930] + demand_range_mw: null + sources: [source:eia:eia930] + notes: "Largest of the three major North American interconnections; Rockies to the Atlantic including most of eastern Canada, excluding Quebec and Texas." + + - grid_id: grid:us-western + name: Western Interconnection + kind: interconnection + parent_grid: null + countries: [us, ca, mx] + frequency_hz: 60 + timezone: America/Denver + operators: + - { name: Western Electricity Coordinating Council, role: regulator } + market: null + interconnections: + - { to_grid: grid:us-eastern, name: "back-to-back HVDC ties", capacity_mw: 1300 } + live_data: + availability: partial + notes: hourly balancing-authority-level demand, generation and interchange via EIA-930 + source_ids: [source:eia:eia930] + demand_range_mw: null + sources: [source:eia:eia930] + notes: "Spans 14 western US states, Alberta, British Columbia and Baja California (Mexico); CAISO's WEIM/EDAM markets increasingly span the footprint." + + - grid_id: grid:us-texas + name: Texas Interconnection + kind: interconnection + parent_grid: null + countries: [us] + frequency_hz: 60 + timezone: America/Chicago + operators: + - { name: Texas Reliability Entity, role: regulator } + market: null + interconnections: + - { to_grid: grid:us-eastern, name: "DC ties (North and East)", capacity_mw: 820 } + - { to_grid: grid:mx-sin, name: "DC ties and VFT (Railroad, Eagle Pass, Laredo)", capacity_mw: 430 } + live_data: + availability: rich + notes: coextensive with ERCOT; real-time public data via ERCOT dashboards and EIA-930 + source_ids: [source:ercot:mis, source:eia:eia930] + demand_range_mw: { min: 28000, max: 86000 } + sources: [source:eia:eia930] + notes: "Electrically separate synchronous system covering most of Texas; asynchronous DC ties only, historically preserving intrastate regulatory status." + + - grid_id: grid:us-pjm + name: PJM Interconnection + kind: iso_rto + parent_grid: grid:us-eastern + countries: [us] + frequency_hz: 60 + timezone: America/New_York + operators: + - { name: PJM Interconnection, role: system_operator } + - { name: PJM Interconnection, role: market_operator } + - { name: Federal Energy Regulatory Commission, role: regulator } + market: + structure: energy, capacity (RPM) and ancillary services markets + price_mechanism: nodal locational marginal pricing, 5-minute real-time settlement + dispatch_interval: 5min + interconnections: [] + live_data: + availability: rich + notes: public real-time LMPs, load and generation mix via Data Miner 2 API + source_ids: [source:pjm:dataminer2, source:eia:eia930] + demand_range_mw: { min: 60000, max: 165000 } + sources: [source:pjm:dataminer2, source:eia:eia930] + notes: "World's largest wholesale power market by volume; 13 mid-Atlantic/Midwest states plus DC; datacenter-driven load growth." + + - grid_id: grid:us-miso + name: Midcontinent Independent System Operator + kind: iso_rto + parent_grid: grid:us-eastern + countries: [us, ca] + frequency_hz: 60 + timezone: America/Chicago + operators: + - { name: Midcontinent Independent System Operator, role: system_operator } + - { name: Midcontinent Independent System Operator, role: market_operator } + - { name: Federal Energy Regulatory Commission, role: regulator } + market: + structure: energy and ancillary markets plus annual capacity auction + price_mechanism: nodal locational marginal pricing, 5-minute real-time settlement + dispatch_interval: 5min + interconnections: [] + live_data: + availability: rich + notes: public real-time LMPs, load and fuel mix via MISO market data APIs + source_ids: [source:miso:market-data, source:eia:eia930] + demand_range_mw: { min: 55000, max: 127000 } + sources: [source:miso:market-data, source:eia:eia930] + notes: "15 US states plus Manitoba, in north and south sub-regions with a constrained transfer path between them." + + - grid_id: grid:us-caiso + name: California Independent System Operator + kind: iso_rto + parent_grid: grid:us-western + countries: [us] + frequency_hz: 60 + timezone: America/Los_Angeles + operators: + - { name: California Independent System Operator, role: system_operator } + - { name: California Independent System Operator, role: market_operator } + - { name: Federal Energy Regulatory Commission, role: regulator } + market: + structure: energy and ancillary markets; Western Energy Imbalance Market and EDAM extend beyond CAISO + price_mechanism: nodal locational marginal pricing, 15-minute market plus 5-minute dispatch + dispatch_interval: 5min + interconnections: [] + live_data: + availability: rich + notes: public real-time prices, demand, generation mix and curtailment via OASIS and Today's Outlook + source_ids: [source:caiso:oasis, source:eia:eia930] + demand_range_mw: { min: 15000, max: 52000 } + sources: [source:caiso:oasis, source:eia:eia930] + notes: "Solar duck-curve archetype; large battery fleet; imports heavily over Pacific interties." + + - grid_id: grid:us-ercot + name: Electric Reliability Council of Texas + kind: iso_rto + parent_grid: grid:us-texas + countries: [us] + frequency_hz: 60 + timezone: America/Chicago + operators: + - { name: Electric Reliability Council of Texas, role: system_operator } + - { name: Electric Reliability Council of Texas, role: market_operator } + - { name: Public Utility Commission of Texas, role: regulator } + market: + structure: energy-only market, no capacity market + price_mechanism: nodal locational marginal pricing with operating reserve demand curve adders, 5-minute SCED + dispatch_interval: 5min + interconnections: [] + live_data: + availability: rich + notes: real-time prices, load, fuel mix and reserves publicly on ERCOT dashboards and MIS + source_ids: [source:ercot:mis, source:eia:eia930] + demand_range_mw: { min: 28000, max: 86000 } + sources: [source:ercot:mis, source:eia:eia930] + notes: "Covers ~90% of Texas load; fastest-growing wind, solar and battery fleet in the US; scarcity-pricing design tested by Winter Storm Uri (2021)." + + - grid_id: grid:us-spp + name: Southwest Power Pool + kind: iso_rto + parent_grid: grid:us-eastern + countries: [us] + frequency_hz: 60 + timezone: America/Chicago + operators: + - { name: Southwest Power Pool, role: system_operator } + - { name: Southwest Power Pool, role: market_operator } + - { name: Federal Energy Regulatory Commission, role: regulator } + market: + structure: Integrated Marketplace energy and ancillary markets + price_mechanism: nodal locational marginal pricing, 5-minute real-time settlement + dispatch_interval: 5min + interconnections: [] + live_data: + availability: rich + notes: public real-time prices and operational data via SPP Marketplace portal; hourly via EIA-930 + source_ids: [source:eia:eia930] + demand_range_mw: { min: 20000, max: 56000 } + sources: [source:eia:eia930] + notes: "14-state footprint on the wind-rich Great Plains; frequently very high instantaneous wind penetration; expanding into the Western Interconnection with Markets+." + + - grid_id: grid:us-nyiso + name: New York Independent System Operator + kind: iso_rto + parent_grid: grid:us-eastern + countries: [us] + frequency_hz: 60 + timezone: America/New_York + operators: + - { name: New York Independent System Operator, role: system_operator } + - { name: New York Independent System Operator, role: market_operator } + - { name: Federal Energy Regulatory Commission, role: regulator } + market: + structure: energy, capacity (ICAP) and ancillary markets + price_mechanism: zonal/nodal locational based marginal pricing, 5-minute real-time settlement + dispatch_interval: 5min + interconnections: [] + live_data: + availability: rich + notes: public real-time LBMPs, load and fuel mix via NYISO OASIS and dashboards + source_ids: [source:eia:eia930] + demand_range_mw: { min: 14000, max: 34000 } + sources: [source:eia:eia930] + notes: "Single-state ISO; chronic upstate-downstate transmission constraint; Champlain Hudson HVDC bringing Quebec hydro to NYC." + + - grid_id: grid:us-isone + name: ISO New England + kind: iso_rto + parent_grid: grid:us-eastern + countries: [us] + frequency_hz: 60 + timezone: America/New_York + operators: + - { name: ISO New England, role: system_operator } + - { name: ISO New England, role: market_operator } + - { name: Federal Energy Regulatory Commission, role: regulator } + market: + structure: energy, forward capacity and ancillary markets + price_mechanism: nodal locational marginal pricing, 5-minute real-time settlement + dispatch_interval: 5min + interconnections: [] + live_data: + availability: rich + notes: public real-time prices, load and fuel mix via ISO Express API and dashboards + source_ids: [source:eia:eia930] + demand_range_mw: { min: 9000, max: 28500 } + sources: [source:eia:eia930] + notes: "Six New England states; winter gas-electric fuel security is the defining risk; HVDC imports from Quebec and New Brunswick ties." + + - grid_id: grid:ca-ieso + name: Independent Electricity System Operator (Ontario) + kind: iso_rto + parent_grid: grid:us-eastern + countries: [ca] + frequency_hz: 60 + timezone: America/Toronto + operators: + - { name: Independent Electricity System Operator, role: system_operator } + - { name: Independent Electricity System Operator, role: market_operator } + - { name: Hydro One, role: transmission_owner } + - { name: Ontario Energy Board, role: regulator } + market: + structure: single-schedule market since the May 2025 market renewal + price_mechanism: locational marginal pricing, 5-minute dispatch + dispatch_interval: 5min + interconnections: [] + live_data: + availability: rich + notes: 5-minute public demand, price, generator output and intertie flow via IESO public reports + source_ids: [source:ieso:public-reports] + demand_range_mw: { min: 11000, max: 27000 } + sources: [source:ieso:public-reports] + notes: "Nuclear-hydro dominated; synchronous within the Eastern Interconnection with ties to Manitoba, Michigan, New York and Minnesota; radial/asynchronous ties toward Quebec." + + - grid_id: grid:ca-aeso + name: Alberta Electric System Operator + kind: iso_rto + parent_grid: grid:us-western + countries: [ca] + frequency_hz: 60 + timezone: America/Edmonton + operators: + - { name: Alberta Electric System Operator, role: system_operator } + - { name: Alberta Electric System Operator, role: market_operator } + - { name: Alberta Utilities Commission, role: regulator } + market: + structure: energy-only power pool; restructuring toward a day-ahead market under review + price_mechanism: system marginal price averaged to an hourly pool price + dispatch_interval: null + interconnections: [] + live_data: + availability: rich + notes: real-time pool price, demand and per-asset generation via the public AESO ETS current supply-demand report + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 8000, max: 12500 } + sources: [source:wikidata:sparql] + notes: "Synchronous member of the Western Interconnection via the BC tie; also HVDC back-to-back tie to Saskatchewan (Eastern Interconnection); gas-dominated after full coal exit (2024)." + + - grid_id: grid:ca-hydro-quebec + name: Quebec Interconnection (Hydro-Quebec) + kind: synchronous_area + parent_grid: null + countries: [ca] + frequency_hz: 60 + timezone: America/Toronto + operators: + - { name: Hydro-Quebec TransEnergie, role: system_operator } + - { name: Hydro-Quebec TransEnergie, role: transmission_owner } + - { name: Regie de l'energie du Quebec, role: regulator } + market: + structure: vertically integrated; Hydro-Quebec Production dominant, no organized wholesale market + price_mechanism: regulated heritage-pool tariffs; export sales into neighbouring markets + dispatch_interval: null + interconnections: + - { to_grid: grid:us-eastern, name: "HVDC and variable-frequency-transformer ties (New York, New England, Ontario, New Brunswick)", capacity_mw: 5500 } + live_data: + availability: partial + notes: public near-real-time aggregate demand and open-data portal; no dispatch-level feed + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 15000, max: 43000 } + sources: [source:wikidata:sparql] + notes: "One of NERC's four interconnections in its own right: operates asynchronously from the Eastern Interconnection with all external ties HVDC or VFT; vast winter-peaking hydro system (~37 GW hydro)." + + - grid_id: grid:mx-sin + name: Sistema Interconectado Nacional (Mexico) + kind: national_grid + parent_grid: null + countries: [mx] + frequency_hz: 60 + timezone: America/Mexico_City + operators: + - { name: Centro Nacional de Control de Energia (CENACE), role: system_operator } + - { name: Centro Nacional de Control de Energia (CENACE), role: market_operator } + - { name: Comision Federal de Electricidad, role: transmission_owner } + - { name: Comision Nacional de Energia, role: regulator } + market: + structure: short-term wholesale market (MEM) from the 2014 reform; CFE dominant, 2024-25 counter-reform re-centralizing + price_mechanism: nodal local marginal prices, day-ahead and real-time + dispatch_interval: 5min + interconnections: + - { to_grid: grid:us-texas, name: "DC ties and VFT (Railroad, Eagle Pass, Laredo)", capacity_mw: 430 } + - { to_grid: grid:siepac, name: "AC tie to Guatemala (Tapachula-Los Brillantes), normally operated radially", capacity_mw: 200 } + live_data: + availability: partial + notes: CENACE publishes demand, dispatch and nodal price data with modest delay; the real portal is CENACE's market information system + source_ids: [source:wikidata:sparql] + demand_range_mw: { min: 25000, max: 53000 } + sources: [source:wikidata:sparql] + notes: "Own synchronous system; Baja California norte is instead part of the Western Interconnection and Baja California Sur is isolated; asynchronous ties to ERCOT." diff --git a/intelligence/schemas/grid.schema.json b/intelligence/schemas/grid.schema.json index 0c41a05..4c89803 100644 --- a/intelligence/schemas/grid.schema.json +++ b/intelligence/schemas/grid.schema.json @@ -4,13 +4,23 @@ "title": "Grid registry record", "description": "Registry record for a power grid / synchronous area / market (richer than a Grid entity).", "type": "object", - "required": ["grid_id", "name", "kind", "countries", "frequency_hz", "live_data", "sources"], + "required": [ + "grid_id", + "name", + "kind", + "countries", + "frequency_hz", + "live_data", + "sources" + ], "properties": { "grid_id": { "type": "string", "pattern": "^grid:[a-z0-9-]+$" }, - "name": { "type": "string" }, + "name": { + "type": "string" + }, "kind": { "enum": [ "synchronous_area", @@ -27,45 +37,81 @@ }, "countries": { "type": "array", - "items": { "$ref": "defs.schema.json#/$defs/countryCode" }, + "items": { + "$ref": "defs.schema.json#/$defs/countryCode" + }, "minItems": 1 }, - "frequency_hz": { "enum": [50, 60] }, + "frequency_hz": { + "enum": [ + 50, + 60 + ] + }, "live_data": { "type": "object", - "required": ["availability"], + "required": [ + "availability" + ], "properties": { "availability": { - "enum": ["rich", "partial", "minimal", "none"] + "enum": [ + "rich", + "partial", + "minimal", + "none" + ] + }, + "notes": { + "type": "string" }, - "notes": { "type": "string" }, "source_ids": { "type": "array", - "items": { "type": "string" } + "items": { + "type": "string" + } } }, "additionalProperties": false }, "sources": { "type": "array", - "items": { "type": "string" }, + "items": { + "type": "string" + }, "minItems": 1 }, "parent_grid": { "anyOf": [ - { "type": "string", "pattern": "^grid:[a-z0-9-]+$" }, - { "type": "null" } + { + "type": "string", + "pattern": "^grid:[a-z0-9-]+$" + }, + { + "type": "null" + } ] }, "operators": { "type": "array", "items": { "type": "object", - "required": ["name", "role"], + "required": [ + "name", + "role" + ], "properties": { - "name": { "type": "string" }, + "name": { + "type": "string" + }, "role": { - "enum": ["system_operator", "market_operator", "tso", "transmission_owner", "regulator"] + "enum": [ + "system_operator", + "market_operator", + "tso", + "transmission_owner", + "regulator" + ] } }, "additionalProperties": false @@ -76,26 +122,54 @@ { "type": "object", "properties": { - "structure": { "type": "string" }, - "price_mechanism": { "type": "string" }, - "dispatch_interval": { "type": "string" } + "structure": { + "type": [ + "string", + "null" + ] + }, + "price_mechanism": { + "type": [ + "string", + "null" + ] + }, + "dispatch_interval": { + "type": [ + "string", + "null" + ] + } }, "additionalProperties": false }, - { "type": "null" } + { + "type": "null" + } ] }, - "timezone": { "type": "string" }, + "timezone": { + "type": "string" + }, "interconnections": { "type": "array", "items": { "type": "object", - "required": ["to_grid"], + "required": [ + "to_grid" + ], "properties": { - "to_grid": { "type": "string" }, - "name": { "type": "string" }, + "to_grid": { + "type": "string" + }, + "name": { + "type": "string" + }, "capacity_mw": { - "type": ["number", "null"] + "type": [ + "number", + "null" + ] } }, "additionalProperties": false @@ -106,15 +180,23 @@ { "type": "object", "properties": { - "min": { "type": "number" }, - "max": { "type": "number" } + "min": { + "type": "number" + }, + "max": { + "type": "number" + } }, "additionalProperties": false }, - { "type": "null" } + { + "type": "null" + } ] }, - "notes": { "type": "string" } + "notes": { + "type": "string" + } }, "additionalProperties": false -} +} \ No newline at end of file From 259340461d002fbcd470808f4137f2f3ba3f6f60 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 01:52:40 +0000 Subject: [PATCH 09/16] =?UTF-8?q?feat(intelligence):=20global=20source=20r?= =?UTF-8?q?egistry=20=E2=80=94=2067=20data=20sources=20(phases=202-3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AEMO deep coverage (NEMWEB report families, dashboard API, MMS data model, Gas BB, WEM) plus global breadth: North American ISOs, European TSOs and platforms, Asian operators, maritime AIS providers with licensing verdicts (aisstream free tier vs AISHub reciprocity vs commercial-only aggregators), weather, trade and satellite datasets. Every record carries verification_status: documented_only — this environment cannot reach the endpoints, and no live verification is claimed. Licensing vocabulary widened to capture real terms (public-domain, open-noncommercial, reciprocal, mixed). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014yVGqBtvM1zNAd9tvUVNzm --- .../data/registries/sources/aemo.yaml | 270 +++++++++++++++ .../data/registries/sources/asia-other.yaml | 163 +++++++++ .../registries/sources/australia-other.yaml | 132 ++++++++ .../data/registries/sources/europe.yaml | 241 ++++++++++++++ .../registries/sources/global-datasets.yaml | 313 ++++++++++++++++++ .../data/registries/sources/maritime.yaml | 261 +++++++++++++++ .../registries/sources/north-america.yaml | 296 +++++++++++++++++ .../data/registries/sources/weather.yaml | 105 ++++++ intelligence/schemas/source.schema.json | 144 ++++++-- 9 files changed, 1897 insertions(+), 28 deletions(-) create mode 100644 intelligence/data/registries/sources/aemo.yaml create mode 100644 intelligence/data/registries/sources/asia-other.yaml create mode 100644 intelligence/data/registries/sources/australia-other.yaml create mode 100644 intelligence/data/registries/sources/europe.yaml create mode 100644 intelligence/data/registries/sources/global-datasets.yaml create mode 100644 intelligence/data/registries/sources/maritime.yaml create mode 100644 intelligence/data/registries/sources/north-america.yaml create mode 100644 intelligence/data/registries/sources/weather.yaml diff --git a/intelligence/data/registries/sources/aemo.yaml b/intelligence/data/registries/sources/aemo.yaml new file mode 100644 index 0000000..d6dba4d --- /dev/null +++ b/intelligence/data/registries/sources/aemo.yaml @@ -0,0 +1,270 @@ +# Source registry — AEMO (Australian Energy Market Operator). See ../../schemas/source.schema.json +records: + - source_id: source:aemo:nemweb-dispatchis + publisher: Australian Energy Market Operator + dataset: NEMWEB Current DispatchIS Reports + api_type: file-drop + url: https://nemweb.com.au/Reports/Current/DispatchIS_Reports/ + docs_url: https://nemweb.com.au/ + coverage: + grids: [grid:au-nem] + countries: [au] + entity_types: [Grid, GridRegion, Interconnector] + metrics: [grid.demand.mw, grid.generation.mw, market.price.energy, intercon.flow.mw, market.price.fcas] + data_class: live + latency: ~1min after each 5-min dispatch run + resolution: 5min + update_frequency: every 5 minutes + history_depth: rolling ~24h in Current; deeper in Archive and MMSDM + authentication: none + licensing: + class: open-attribution + notes: AEMO copyright permits use of NEM data with attribution for informational purposes per AEMO's copyright permissions. + cost: free + reliability: 5 + quality_notes: Authoritative market operator dispatch solution (prices, regional demand/generation, interconnector flows); corrections arrive as re-runs (RUNNO, INTERVENTION flags). + verification_status: documented_only + last_checked: "2026-08-12" + notes: ZIP files containing AEMO CSV C/I/D row format per the MMS Data Model (DISPATCH package - PRICE, REGIONSUM, INTERCONNECTORRES tables). + + - source_id: source:aemo:nemweb-dispatch-scada + publisher: Australian Energy Market Operator + dataset: NEMWEB Current Dispatch_SCADA (unit-level SCADA MW) + api_type: file-drop + url: https://nemweb.com.au/Reports/Current/Dispatch_SCADA/ + docs_url: https://nemweb.com.au/ + coverage: + grids: [grid:au-nem] + countries: [au] + entity_types: [Generator, Storage] + metrics: [power.output.mw] + data_class: live + latency: ~1min after each 5-min dispatch interval + resolution: 5min + update_frequency: every 5 minutes + history_depth: rolling ~24h in Current; deeper in Archive and MMSDM + authentication: none + licensing: + class: open-attribution + notes: AEMO copyright permits use with attribution. + cost: free + reliability: 5 + quality_notes: Actual instantaneous SCADA MW per DUID for every scheduled/semi-scheduled unit - the primary unit-level live generation signal for the NEM. + verification_status: documented_only + last_checked: "2026-08-12" + notes: ZIP of CSV (DISPATCH_UNIT_SCADA); keyed by DUID; join to the NEM Registration and Exemption List for station/fuel metadata. + + - source_id: source:aemo:nemweb-tradingis + publisher: Australian Energy Market Operator + dataset: NEMWEB Current TradingIS Reports (trading/settlement prices) + api_type: file-drop + url: https://nemweb.com.au/Reports/Current/TradingIS_Reports/ + docs_url: https://nemweb.com.au/ + coverage: + grids: [grid:au-nem] + countries: [au] + entity_types: [Grid, GridRegion, Interconnector] + metrics: [market.price.energy, grid.demand.mw, intercon.flow.mw] + data_class: live + latency: ~minutes after each interval + resolution: 5min + update_frequency: every 5 minutes (5-minute settlement since Oct 2021) + history_depth: rolling ~24h in Current; deeper in Archive and MMSDM + authentication: none + licensing: + class: open-attribution + notes: AEMO copyright permits use with attribution. + cost: free + reliability: 5 + quality_notes: Settlement-relevant trading prices per region; since five-minute settlement (1 Oct 2021) trading price equals the 5-min dispatch price, so DispatchIS and TradingIS have converged in cadence. + verification_status: documented_only + last_checked: "2026-08-12" + notes: ZIP of CSV (TRADING package - PRICE, REGIONSUM, INTERCONNECTORRES). + + - source_id: source:aemo:nemweb-next-day-dispatch + publisher: Australian Energy Market Operator + dataset: NEMWEB Next_Day_Dispatch (full unit dispatch detail, next-day publication) + api_type: file-drop + url: https://nemweb.com.au/Reports/Current/Next_Day_Dispatch/ + docs_url: https://nemweb.com.au/ + coverage: + grids: [grid:au-nem] + countries: [au] + entity_types: [Generator, Storage, GridRegion] + metrics: [power.output.mw, dispatch.target.mw, dispatch.availability.mw] + data_class: live + latency: published ~04:00 AEST the following trading day + resolution: 5min + update_frequency: daily + history_depth: rolling window in Current; deeper in Archive and MMSDM + authentication: none + licensing: + class: open-attribution + notes: AEMO copyright permits use with attribution. + cost: free + reliability: 5 + quality_notes: Next-day disclosure of full unit-level dispatch targets, availabilities and bid bands (DISPATCHLOAD etc.) - richer than real-time SCADA but delayed one day. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Complements Dispatch_SCADA for reconciliation and detecting constrained-on/off behaviour. + + - source_id: source:aemo:vis-nem-summary + publisher: Australian Energy Market Operator + dataset: AEMO visualisations API - ELEC_NEM_SUMMARY (NEM data dashboard JSON) + api_type: rest + url: https://visualisations.aemo.com.au/aemo/apps/api/report/ELEC_NEM_SUMMARY + docs_url: https://aemo.com.au/energy-systems/electricity/national-electricity-market-nem/data-nem/data-dashboard-nem + coverage: + grids: [grid:au-nem] + countries: [au] + entity_types: [Grid, GridRegion, Interconnector] + metrics: [grid.demand.mw, grid.generation.mw, market.price.energy, intercon.flow.mw] + data_class: live + latency: seconds to ~1min behind dispatch + resolution: 5min + update_frequency: every 5 minutes + history_depth: current snapshot only (no history at this endpoint) + authentication: none + licensing: + class: open-attribution + notes: Backing service for AEMO's public NEM data dashboard; AEMO copyright with attribution. Not a formally documented public API - subject to change without notice. + cost: free + reliability: 4 + quality_notes: Convenient single-call JSON summary (per-region price/demand/generation, interconnector flows); same underlying data as DispatchIS but an undocumented dashboard endpoint - treat NEMWEB as canonical. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Sibling reports include 5MIN and ELEC_NEM_SUMMARY_PRICES on the same /aemo/apps/api/report/ base. + + - source_id: source:aemo:nem-registration-and-exemption-list + publisher: Australian Energy Market Operator + dataset: NEM Registration and Exemption List (registered participants and generating units) + api_type: bulk-download + url: https://aemo.com.au/energy-systems/electricity/national-electricity-market-nem/participate-in-the-market/registration + docs_url: https://aemo.com.au/energy-systems/electricity/national-electricity-market-nem/participate-in-the-market/registration + coverage: + grids: [grid:au-nem] + countries: [au] + entity_types: [Generator, Storage, Company] + metrics: [asset.capacity.mw] + data_class: static + latency: n/a (structural reference data) + resolution: unit-level (DUID / station) + update_frequency: updated as registrations change (roughly monthly in practice) + history_depth: current snapshot; no published version history + authentication: none + licensing: + class: open-attribution + notes: AEMO copyright permits use with attribution. + cost: free + reliability: 5 + quality_notes: Authoritative DUID-to-station mapping - registered capacity, fuel source, technology type, dispatch type, region, participant. The join key for all NEM unit-level telemetry. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Published as an XLSX workbook ("NEM Registration and Exemption List.xlsx") with multiple tabs; sheet 'PU and Scheduled Loads' carries generating unit detail. + + - source_id: source:aemo:mms-data-model + publisher: Australian Energy Market Operator + dataset: MMS Data Model documentation + MMSDM historical archive + api_type: bulk-download + url: https://nemweb.com.au/Data_Archive/Wholesale_Electricity/MMSDM/ + docs_url: https://nemweb.com.au/Reports/Current/MMSDataModelReport/ + coverage: + grids: [grid:au-nem] + countries: [au] + entity_types: [Grid, GridRegion, Generator, Interconnector] + metrics: [power.output.mw, grid.demand.mw, market.price.energy, intercon.flow.mw] + data_class: both + latency: monthly archive lags ~1 month; documentation static + resolution: 5min and coarser depending on table + update_frequency: monthly archive drops; data model docs revised per release + history_depth: complete NEM history back to market start (Dec 1998) + authentication: none + licensing: + class: open-attribution + notes: AEMO copyright permits use with attribution. + cost: free + reliability: 5 + quality_notes: The schema bible for every NEMWEB CSV (table/column definitions, keys, C/I/D row conventions) plus the complete monthly historical archive - the backfill path for all NEM telemetry. + verification_status: documented_only + last_checked: "2026-08-12" + notes: MMSDM monthly ZIPs are large (GB-scale); data model report defines every package/table used by the dispatchis/scada/tradingis connectors. + + - source_id: source:aemo:gas-bulletin-board + publisher: Australian Energy Market Operator + dataset: Gas Bulletin Board (GBB) - east coast gas flows, capacity outlooks, notices + api_type: file-drop + url: https://nemweb.com.au/Reports/Current/GBB/ + docs_url: https://www.aemo.com.au/energy-systems/gas/gas-bulletin-board-gbb/data-gbb + coverage: + countries: [au] + entity_types: [Pipeline, GasPlant, Storage, Facility] + metrics: [gas.pipeline.flow.tj_day, gas.production.tj_day, gas.storage.level.tj, gas.capacity.outlook.tj_day] + data_class: live + latency: actual flows typically published within ~30min of receipt; daily D+1 for most actuals + resolution: daily (TJ/day) per facility/connection point + update_frequency: intraday updates as operators submit; daily consolidated reports + history_depth: last 31 days in current reports; full history via historical GBB reports + authentication: none + licensing: + class: open-attribution + notes: AEMO copyright permits use with attribution; GBB data submitted by facility operators under the National Gas Rules. + cost: free + reliability: 4 + quality_notes: Covers all major production fields, transmission pipelines, storage and demand centres in eastern/northern Australia; operator-submitted, so occasional resubmissions/corrections. + verification_status: documented_only + last_checked: "2026-08-12" + notes: CSV reports (Actual Flows and Storage, Capacity Outlook, Medium Term Capacity, Notices); WA has a separate WA GBB (gbbwa.aemo.com.au). + + - source_id: source:aemo:wem-data-portal + publisher: Australian Energy Market Operator + dataset: WEM (Wholesale Electricity Market, WA/SWIS) public market data + api_type: file-drop + url: https://data.wa.aemo.com.au/ + docs_url: https://www.aemo.com.au/energy-systems/electricity/wholesale-electricity-market-wem/data-wem + coverage: + grids: [grid:au-swis] + countries: [au] + entity_types: [Grid, Generator, Company] + metrics: [grid.demand.mw, power.output.mw, market.price.energy, market.price.stem] + data_class: both + latency: near-real-time for balancing/load files; daily-to-monthly for others + resolution: 5min (post-reform dispatch) / 30min trading intervals; varies by dataset + update_frequency: varies by file - from every dispatch interval to monthly + history_depth: multi-year CSV history for most datasets; ~1 day of JSON data-model snapshots + authentication: none + licensing: + class: open-attribution + notes: AEMO copyright permits use with attribution. + cost: free + reliability: 4 + quality_notes: The SWIS is islanded from the NEM, so this portal (not NEMWEB) is the source of truth for WA; facility-level SCADA, balancing summary, STEM prices, outages, load forecasts. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Plain CSV/JSON directory trees under /public/; WEM reform (Oct 2023) split datasets into pre/post-reform series - handle the discontinuity. + + - source_id: source:aemo:isp-inputs-assumptions + publisher: Australian Energy Market Operator + dataset: Integrated System Plan (ISP) - inputs, assumptions and scenarios workbooks + api_type: bulk-download + url: https://www.aemo.com.au/energy-systems/major-publications/integrated-system-plan-isp/2026-integrated-system-plan-isp + docs_url: https://www.aemo.com.au/energy-systems/major-publications/integrated-system-plan-isp + coverage: + grids: [grid:au-nem] + countries: [au] + entity_types: [Grid, GridRegion, Generator, TransmissionLine] + metrics: [asset.capacity.mw, forecast.demand.gwh, forecast.capacity.mw] + data_class: static + latency: n/a (planning publication) + resolution: scenario/trace level (half-hourly demand traces; annual build schedules) + update_frequency: ISP biennial (2026 ISP published); IASR consultation updates in between + history_depth: editions back to 2018 ISP remain published + authentication: none + licensing: + class: open-attribution + notes: AEMO copyright permits use with attribution. + cost: free + reliability: 5 + quality_notes: Forward-looking structural context (retirement schedules, committed projects, transmission options, demand traces) for Step Change / Slower Growth / Accelerated Transition scenarios - not telemetry. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Workbooks (XLSX), demand traces and generation outlook data files accompany the 2025-26 Inputs Assumptions and Scenarios Report (IASR). diff --git a/intelligence/data/registries/sources/asia-other.yaml b/intelligence/data/registries/sources/asia-other.yaml new file mode 100644 index 0000000..2dec306 --- /dev/null +++ b/intelligence/data/registries/sources/asia-other.yaml @@ -0,0 +1,163 @@ +# Source registry — Asia & other regions (Japan, India, Korea, Brazil, South Africa). See ../../schemas/source.schema.json +records: + - source_id: source:occto:web + publisher: OCCTO (Organization for Cross-regional Coordination of Transmission Operators, Japan) + dataset: OCCTO wide-area supply-demand and interconnector data (system information service) + api_type: html + url: https://www.occto.or.jp/en/ + docs_url: https://www.occto.or.jp/en/information_disclosure/ + coverage: + grids: [grid:jp] + countries: [jp] + entity_types: [Grid, GridRegion, Interconnector] + metrics: [grid.demand.mw, grid.generation.mw, intercon.flow.mw, reserve.margin.pct] + data_class: both + latency: cross-regional reserve margin updated every ~30min; area supply-demand actuals published monthly + resolution: 30min supply-demand; 10 service areas + update_frequency: 30min for reserve-margin displays; monthly for consolidated demand/supply data files + history_depth: monthly publications archived for years + authentication: registration + licensing: + class: restricted + notes: Public pages free to view; the detailed system information service (keito joho) requires user registration, and terms are Japanese-language with no open licence. Per-area live demand is more accessibly published by each regional utility's denki yoho (demand forecast) pages. + cost: free + reliability: 3 + quality_notes: The only cross-Japan coordinator view (10 areas + interconnector utilisation), but programmatic access is awkward (Japanese HTML/CSV, registration walls). Treat as reference/context; utility juyo-jisseki CSVs are the live path. + verification_status: documented_only + last_checked: "2026-08-12" + notes: English pages summarise; operational data is mostly on the Japanese site. Occtonet (occtonet.occto.or.jp) hosts the member data system. + + - source_id: source:jepx:spot-market + publisher: Japan Electric Power Exchange (JEPX) + dataset: JEPX day-ahead (spot) and intraday market price/volume CSVs + api_type: bulk-download + url: https://www.jepx.jp/electricpower/market-data/spot/ + docs_url: https://www.jepx.jp/en/ + coverage: + grids: [grid:jp] + countries: [jp] + entity_types: [Grid, GridRegion] + metrics: [market.price.energy, market.volume.mwh] + data_class: both + latency: day-ahead results published ~10:00 JST daily for next-day delivery + resolution: 30min products, 9 price areas + system price + update_frequency: daily (spot); continuous session files (intraday) + history_depth: annual CSVs back to FY2005 (spot) + authentication: none + licensing: + class: open-attribution + notes: CSVs are freely downloadable from the JEPX site; JEPX terms request source attribution and prohibit commercial redistribution of the files as-is. No English machine interface - column headers are Japanese. + cost: free + reliability: 4 + quality_notes: Authoritative Japanese wholesale spot prices (~30-40 percent of Japanese demand trades through JEPX). Simple stable CSV URLs (spot_YYYY.csv pattern) - no API needed. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Area prices map to the 9 utility service areas (Hokkaido...Kyushu); Okinawa is not interconnected and has no JEPX area. + + - source_id: source:gridindia:psp-reports + publisher: Grid Controller of India (Grid-India) / National Load Despatch Centre + dataset: Daily Power Supply Position (PSP) reports and NLDC operational reports + api_type: file-drop + url: https://report.grid-india.in/psp_report.php + docs_url: https://grid-india.in/en + coverage: + grids: [grid:in] + countries: [in] + entity_types: [Grid, GridRegion] + metrics: [grid.demand.mw, grid.demand.met.mu, energy.shortage.mu, grid.frequency.hz] + data_class: both + latency: daily report published next day + resolution: daily aggregates by state/region; peak demand met, energy met/shortage, frequency profile + update_frequency: daily (PSP); weekly/monthly consolidated reports + history_depth: archive back many years (posoco.in legacy + report.grid-india.in) + authentication: none + licensing: + class: open-attribution + notes: Government publication, free to download; no explicit open-data licence stated - cite Grid-India. Reports are PDF and XLS. + cost: free + reliability: 3 + quality_notes: Authoritative all-India operational summary, but file-based (PDF/XLS parsing) and daily granularity only. Real-time all-India demand is visible on the Grid-India dashboard and via Merit/Vidyut Pravah portals. + verification_status: documented_only + last_checked: "2026-08-12" + notes: XLS variant of the daily PSP is machine-readable; formats have changed over the years - version the parser. + + - source_id: source:kpx:epsis + publisher: Korea Power Exchange (KPX) + dataset: EPSIS - Electric Power Statistics Information System (plus data.go.kr open APIs) + api_type: html + url: https://epsis.kpx.or.kr/ + docs_url: https://epsis.kpx.or.kr/epsisnew/selectEkccIntroEn.do?menuId=090101 + coverage: + grids: [grid:kr] + countries: [kr] + entity_types: [Grid, Generator, Company] + metrics: [grid.demand.mw, grid.generation.mw, market.price.energy, asset.capacity.mw] + data_class: both + latency: statistics lag days-to-months; real-time SMP/demand on KPX site + resolution: hourly (SMP, demand) to annual statistics (~700 items, 7 categories) + update_frequency: hourly market series; monthly/annual statistics + history_depth: statistical series back to 1961 + authentication: none + licensing: + class: open-registration + notes: EPSIS web access is free; downloadable tables free with attribution. Programmatic access is via Korea's public data portal (data.go.kr) open APIs, which require free registration and a service key. + cost: free + reliability: 3 + quality_notes: Deep official statistics (KEPCO + KPX sourced); English UI exists but data labels are mixed-language. SMP and load are authoritative; site is form-driven (POST) rather than a clean API. + verification_status: documented_only + last_checked: "2026-08-12" + notes: For live use, prefer the data.go.kr OpenAPI endpoints for SMP/demand and treat EPSIS as the statistical archive. + + - source_id: source:ons-brazil:open-data + publisher: ONS (Operador Nacional do Sistema Elétrico, Brazil) + dataset: ONS Dados Abertos - open data portal for the Brazilian interconnected system (SIN) + api_type: rest + url: https://dados.ons.org.br/ + docs_url: https://dados.ons.org.br/dataset/ + coverage: + grids: [grid:br-sin] + countries: [br] + entity_types: [Grid, GridRegion, Generator, Reservoir, Interconnector] + metrics: [grid.demand.mw, grid.generation.mw, power.output.mw, storage.level.pct, intercon.flow.mw] + data_class: both + latency: most datasets daily; verified load published D+1; some hourly series + resolution: hourly/half-hourly for load and generation; daily for reservoir levels (EAR) + update_frequency: daily for operational series; some near-real-time + history_depth: multi-year to multi-decade (hydrological series especially) + authentication: none + licensing: + class: open-attribution + notes: Open portal, no login required; datasets carry open licences (ODbL/CC-style per dataset) - attribute ONS. Also mirrored on the AWS Open Data Registry. The separate API Portal (portal-integra) is for SINtegre agents only. + cost: free + reliability: 4 + quality_notes: Excellent coverage of a hydro-dominated system - reservoir stored energy (EAR/ENA) by subsystem, per-plant generation, verified load, interchange. CKAN with CSV/Parquet resources. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Portuguese-language field names; subsystems SE/CO, S, NE, N are the regional keys. + + - source_id: source:eskom:data-portal + publisher: Eskom Holdings SOC Ltd (South Africa) + dataset: Eskom Data Portal (system adequacy, demand, generation performance, renewables) + api_type: bulk-download + url: https://www.eskom.co.za/dataportal/ + docs_url: https://www.eskom.co.za/dataportal/data-request-form/ + coverage: + grids: [grid:za] + countries: [za] + entity_types: [Grid, Generator] + metrics: [grid.demand.mw, grid.generation.mw, capacity.available.mw, load_shedding.stage] + data_class: both + latency: short-term datasets updated several times per week; long-term dumps on request + resolution: hourly (system demand/available capacity); daily/weekly aggregates + update_frequency: several times per week for the live portal; weekly system status reports + history_depth: up to 5 years per download request + authentication: none + licensing: + class: open-attribution + notes: Free public portal; bulk historical downloads go through a data request form (select datasets, max 5 years). No formal open licence text - attribute Eskom. + cost: free + reliability: 3 + quality_notes: The only substantive public feed for the South African grid (Eskom is a vertically integrated utility, not an ISO). Hourly demand, OCGT usage, EAF, renewables and load-shedding data; portal availability has historically been patchy. + verification_status: documented_only + last_checked: "2026-08-12" + notes: CSV downloads; 'Station_Build_Up' style files carry generation by source. No documented REST API. diff --git a/intelligence/data/registries/sources/australia-other.yaml b/intelligence/data/registries/sources/australia-other.yaml new file mode 100644 index 0000000..42ced86 --- /dev/null +++ b/intelligence/data/registries/sources/australia-other.yaml @@ -0,0 +1,132 @@ +# Source registry — Australia (non-AEMO). See ../../schemas/source.schema.json +records: + - source_id: source:openelectricity:api + publisher: OpenElectricity (formerly OpenNEM; The Superpower Institute) + dataset: OpenElectricity API - curated Australian electricity market data + api_type: rest + url: https://api.openelectricity.org.au/ + docs_url: https://docs.openelectricity.org.au/ + coverage: + grids: [grid:au-nem, grid:au-swis] + countries: [au] + entity_types: [Grid, GridRegion, Generator, Facility] + metrics: [power.output.mw, grid.demand.mw, market.price.energy, emissions.rate.tco2e] + data_class: both + latency: minutes behind AEMO source data + resolution: 5min native; aggregations to hourly/daily/monthly + update_frequency: continuous (tracks AEMO 5-min cadence) + history_depth: full NEM history (1998+) and WEM history, cleaned and facility-mapped + authentication: api_key + licensing: + class: open-attribution + notes: Platform data published under CC BY 4.0; underlying AEMO data under AEMO attribution terms. API key issued free via platform.openelectricity.org.au with rate-limited free tier. + cost: free tier; paid tiers for higher limits + reliability: 4 + quality_notes: Best-in-class cleaned/harmonised layer over NEMWEB and WEM data with stable facility IDs and fuel-tech mapping; third-party (not the market operator), so treat AEMO as canonical for disputes. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Official TypeScript and Python clients (opennem/openelectricity-*); set OPENELECTRICITY_API_KEY. + + - source_id: source:artc:hunter-valley-network + publisher: Australian Rail Track Corporation (ARTC) + dataset: Hunter Valley coal network publications (corridor capacity strategy, network performance reporting) + api_type: html + url: https://www.artc.com.au/about/hv-strategy/ + docs_url: https://www.artc.com.au/ + coverage: + countries: [au] + entity_types: [RailNetwork, RailCorridor] + metrics: [rail.capacity.mtpa, rail.trains.per_day] + data_class: static + latency: n/a (periodic publications) + resolution: corridor/zone level + update_frequency: annual (Hunter Valley Corridor Capacity Strategy); monthly performance reporting goes to the Rail Capacity Group, not the public + history_depth: strategy documents back to ~2010 remain online + authentication: none + licensing: + class: restricted + notes: Standard website copyright; documents are publicly downloadable PDFs but no open-data licence is stated. Operational KPI reporting is provided to industry (RCG/HVCCC), not published openly. + cost: free + reliability: 3 + quality_notes: No live public feed - this is structural/capacity context for the world's largest coal export rail chain (~250 trains/day incl. ~half coal). Live coal-chain throughput is an observability gap; HVCCC publishes limited aggregate performance data. + verification_status: documented_only + last_checked: "2026-08-12" + notes: PDF documents. Pair with Port of Newcastle trade reports for chain throughput ground truth. + + - source_id: source:aurizon:investor-reports + publisher: Aurizon Holdings Limited (ASX:AZJ) + dataset: Investor and operational reports - above-rail coal haulage volumes + api_type: html + url: https://www.aurizon.com.au/investors + docs_url: https://www.aurizon.com.au/investors + coverage: + countries: [au] + entity_types: [Company, RailNetwork] + metrics: [rail.haulage.mt, rail.network.tonnes.mt] + data_class: static + latency: weeks after period end + resolution: half-yearly / annual (segment-level tonnages; occasional quarterly updates) + update_frequency: half-yearly results (Feb/Aug) plus ASX announcements + history_depth: ASX filings back to 2010 listing + authentication: none + licensing: + class: restricted + notes: Public ASX disclosure documents; copyright Aurizon. Facts (tonnages) usable; documents not redistributable. + cost: free + reliability: 3 + quality_notes: The dominant QLD coal hauler (~190Mt+/yr across CQCN and NSW); volumes are audited but coarse-grained and lagged - context/calibration data, not telemetry. + verification_status: documented_only + last_checked: "2026-08-12" + notes: PDF results packs; Central Queensland Coal Network (below-rail) throughput also reported. FY2026 results due 17 Aug 2026. + + - source_id: source:qld-gov:coal-industry-statistics + publisher: Queensland Government (Department of Natural Resources and Mines / Resources) + dataset: Queensland coal industry statistics (quarterly coal reports; annual coal statistics; coal industry review tables) + api_type: bulk-download + url: https://www.data.qld.gov.au/dataset/coal-industry-review-statistical-tables + docs_url: https://www.data.qld.gov.au/dataset/quarterly-coal-reports + coverage: + countries: [au] + entity_types: [Mine, Port, Company] + metrics: [production.saleable.mt, production.raw.mt, export.tonnes.mt, stockpile.balance.mt] + data_class: static + latency: ~2-3 months after quarter end + resolution: per-mine (annual review) and aggregate (quarterly); exports by port and destination country + update_frequency: quarterly and annual releases + history_depth: multi-decade series in the coal industry review tables + authentication: none + licensing: + class: open-attribution + notes: Creative Commons Attribution 4.0 (Queensland Government Open Data Portal standard licence). + cost: free + reliability: 4 + quality_notes: The authoritative per-mine saleable/raw production and per-port export series for Queensland - the calibration backbone for any inferred mine-to-port coal flow model. + verification_status: documented_only + last_checked: "2026-08-12" + notes: XLSX/CSV resources on a CKAN portal (data.qld.gov.au) - machine-readable and stable resource URLs. + + - source_id: source:disr:resources-energy-quarterly + publisher: Australian Department of Industry, Science and Resources (Office of the Chief Economist) + dataset: Resources and Energy Quarterly (REQ) - production, export volumes/values and forecasts + api_type: bulk-download + url: https://www.industry.gov.au/publications/resources-and-energy-quarterly + docs_url: https://www.industry.gov.au/publications/resources-and-energy-quarterly + coverage: + countries: [au] + entity_types: [Commodity, Market] + metrics: [export.tonnes.mt, export.value.aud, production.rate.mtpa, price.commodity.usd] + data_class: static + latency: published quarterly (Mar/Jun/Sep/Dec) + resolution: commodity x country, quarterly/annual with 2-year forecasts + update_frequency: quarterly + history_depth: editions back to ~2011; underlying ABS trade series much deeper + authentication: none + licensing: + class: open-attribution + notes: Creative Commons Attribution 4.0 International (standard Commonwealth publication licence). + cost: free + reliability: 4 + quality_notes: The Australian Government's official commodity outlook - iron ore, met/thermal coal, LNG, aluminium chain, lithium etc.; forecast tables in XLSX. Forecasts are scenarios, not observations - keep in derived/context class. + verification_status: documented_only + last_checked: "2026-08-12" + notes: PDF report + data tables workbook per edition. diff --git a/intelligence/data/registries/sources/europe.yaml b/intelligence/data/registries/sources/europe.yaml new file mode 100644 index 0000000..ee03839 --- /dev/null +++ b/intelligence/data/registries/sources/europe.yaml @@ -0,0 +1,241 @@ +# Source registry — Europe. See ../../schemas/source.schema.json +records: + - source_id: source:entsoe:transparency-platform + publisher: ENTSO-E (European Network of Transmission System Operators for Electricity) + dataset: ENTSO-E Transparency Platform (pan-European generation, load, prices, cross-border flows) + api_type: rest + url: https://web-api.tp.entsoe.eu/api + docs_url: https://transparency.entsoe.eu/ + coverage: + countries: [eu] + entity_types: [Grid, GridRegion, Generator, Interconnector] + metrics: [grid.demand.mw, grid.generation.mw, power.output.mw, market.price.energy, intercon.flow.mw] + data_class: both + latency: ~1h for actual generation/load (per Regulation 543/2013 deadlines); day-ahead data on publication + resolution: 15min/30min/60min depending on bidding zone + update_frequency: continuous as TSOs submit; hourly effective cadence for actuals + history_depth: since January 2015 + authentication: registration + licensing: + class: open-registration + notes: Free registration; API access via security token requested by email after signup. Reuse permitted with attribution under the TP terms; some per-unit data carries redistribution caveats. + cost: free + reliability: 4 + quality_notes: The single pan-European source (per-unit actual generation >=100MW, load, cross-border physical flows, day-ahead prices for most zones). TSO-submitted - completeness and timeliness vary notably by country. + verification_status: documented_only + last_checked: "2026-08-12" + notes: XML (IEC 62325-451) responses; EIC codes are the join keys - map to entity external_ids.entsoe_eic. + + - source_id: source:elexon:bmrs + publisher: Elexon (GB Balancing and Settlement Code) + dataset: Elexon Insights Solution API (successor to BMRS - GB balancing mechanism and market data) + api_type: rest + url: https://data.elexon.co.uk/bmrs/api/v1/ + docs_url: https://developer.data.elexon.co.uk/ + coverage: + grids: [grid:gb] + countries: [gb] + entity_types: [Grid, Generator, Interconnector] + metrics: [grid.demand.mw, power.output.mw, grid.frequency.hz, market.price.energy, intercon.flow.mw] + data_class: both + latency: near-real-time (frequency seconds-scale; physical data minutes) + resolution: settlement period (30min) for most; per-BMU physical notifications; instantaneous frequency + update_frequency: continuous; per settlement period for market series + history_depth: multi-year via date-ranged queries; legacy BMRS history preserved + authentication: none + licensing: + class: open-attribution + notes: Insights Solution APIs are public with NO API key required (the legacy BMRS API key regime was retired with the platform). Elexon grants a limited licence to use BMRS data; attribution to Elexon expected. + cost: free + reliability: 4 + quality_notes: Deep GB detail - per-BMU generation (B1610-equivalent), bid/offer acceptances, system prices, demand outturn, frequency. The mandatory ID keeps the familiar 'bmrs' name; the live service is Insights Solution. + verification_status: documented_only + last_checked: "2026-08-12" + notes: JSON/CSV responses; IRIS push service available for streaming delivery (free Azure-based subscription). + + - source_id: source:nordpool:market-data + publisher: Nord Pool (Euronext) + dataset: Nord Pool Data Portal and Power Data Services (day-ahead/intraday prices, volumes, capacities, flows) + api_type: rest + url: https://data.nordpoolgroup.com/ + docs_url: https://www.nordpoolgroup.com/en/services/power-market-data-services/ + coverage: + countries: [no, se, fi, dk, ee, lv, lt, de, nl, be, fr, at, pl, gb] + entity_types: [Grid, GridRegion, Interconnector] + metrics: [market.price.energy, market.volume.mwh, intercon.flow.mw, intercon.capacity.mw] + data_class: both + latency: day-ahead prices published ~12:45-13:00 CET daily; delayed data on the free portal + resolution: hourly (15min products since 2025 MTU change in relevant zones) + update_frequency: daily auction cycle; intraday continuous + history_depth: multi-year on portal; full history via commercial data products + authentication: registration + licensing: + class: commercial + notes: Web Data Portal viewing is free (registration for some views) for personal/internal use, but programmatic API access and redistribution are PAID subscriptions (API accounts ~EUR 350/yr extra, redistribution licences from ~EUR 3500/yr). Academic terms available. Day-ahead prices for most zones are also available free via ENTSO-E TP. + cost: commercial (free delayed web viewing) + reliability: 4 + quality_notes: Primary exchange for Nordic/Baltic day-ahead; authoritative auction results. Use ENTSO-E for a free programmatic path to the same zone prices. + verification_status: documented_only + last_checked: "2026-08-12" + notes: marketdata.nordpoolgroup.com hosts the JSON API for subscribers. + + - source_id: source:neso:data-portal + publisher: National Energy System Operator (NESO, Great Britain) + dataset: NESO Data Portal (CKAN) - demand, forecasts, carbon intensity, balancing, system data + api_type: rest + url: https://www.neso.energy/data-portal + docs_url: https://www.neso.energy/data-portal/about-data-portal + coverage: + grids: [grid:gb] + countries: [gb] + entity_types: [Grid, GridRegion, Interconnector] + metrics: [grid.demand.mw, grid.demand.forecast.mw, emissions.intensity.gco2_kwh, intercon.flow.mw] + data_class: both + latency: minutes-to-hours depending on dataset + resolution: settlement period (30min) typical; some 1min/instantaneous datasets + update_frequency: per dataset - real-time to annual + history_depth: multi-year for most operational datasets + authentication: none + licensing: + class: open-attribution + notes: NESO Open Licence (based on OGL v3.0) on most datasets, applied per dataset - check each dataset's licence field; attribution to NESO required. + cost: free + reliability: 4 + quality_notes: CKAN datastore SQL/API over hundreds of datasets (demand outturn, embedded wind/solar forecasts, carbon intensity, ancillary contracts). Complements Elexon (settlement view) with the operator view. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Formerly National Grid ESO portal (data.nationalgrideso.com redirects); Carbon Intensity API (carbonintensity.org.uk) is a related NESO service. + + - source_id: source:fingrid:open-data + publisher: Fingrid Oyj (Finnish TSO) + dataset: Fingrid Open Data (Finnish power system real-time and historical data) + api_type: rest + url: https://data.fingrid.fi/en + docs_url: https://developer-data.fingrid.fi/apis + coverage: + grids: [grid:fi] + countries: [fi] + entity_types: [Grid, Interconnector] + metrics: [grid.demand.mw, grid.generation.mw, grid.frequency.hz, intercon.flow.mw] + data_class: both + latency: ~3min for real-time datasets + resolution: 3min real-time series; 15min/hourly for market series + update_frequency: every 3 minutes for live datasets + history_depth: multi-year per dataset + authentication: api_key + licensing: + class: open-attribution + notes: CC BY 4.0; free API key on registration via the developer portal (x-api-key header). + cost: free + reliability: 5 + quality_notes: Exceptionally clean open TSO data - real-time production by type, consumption, frequency, cross-border flows (SE/NO/EE/RU links). + verification_status: documented_only + last_checked: "2026-08-12" + notes: Dataset IDs are numeric (e.g. 193 = electricity production in Finland). + + - source_id: source:rte:data-portal + publisher: RTE (Réseau de Transport d'Électricité, French TSO) + dataset: RTE Data Portal APIs + éCO2mix (French generation, consumption, exchanges, CO2) + api_type: rest + url: https://data.rte-france.com/ + docs_url: https://data.rte-france.com/catalog + coverage: + grids: [grid:fr] + countries: [fr] + entity_types: [Grid, GridRegion, Generator, Interconnector] + metrics: [grid.demand.mw, grid.generation.mw, power.output.mw, intercon.flow.mw, emissions.intensity.gco2_kwh] + data_class: both + latency: éCO2mix real-time refreshed ~every 15min-1h; consolidated data later + resolution: 15min (éCO2mix); per-unit actual generation hourly + update_frequency: hourly for most live APIs + history_depth: éCO2mix consolidated history back to 2012 (bulk download); API history varies + authentication: oauth + licensing: + class: open-registration + notes: Free account + application registration; OAuth2 client-credentials token per API. Data reusable with attribution; éCO2mix also republished on ODRÉ (opendatasoft) without auth. API quotas apply (e.g. 50k calls/month for éCO2mix). + cost: free + reliability: 4 + quality_notes: Rich per-unit ('Actual Generation') and national mix data; ODRÉ mirror (odre.opendatasoft.com, eco2mix-national-tr) is the low-friction unauthenticated path. + verification_status: documented_only + last_checked: "2026-08-12" + notes: OAuth2 token endpoint at digital.iservices.rte-france.com; each API subscribed per application. + + - source_id: source:regelleistung:datacenter + publisher: German TSOs (50Hertz, Amprion, TenneT, TransnetBW) via regelleistung.net + dataset: regelleistung.net Datacenter - FCR/aFRR/mFRR balancing capacity and energy tenders + api_type: bulk-download + url: https://www.regelleistung.net/apps/datacenter/tendering-files/ + docs_url: https://www.regelleistung.net/en-us/Data + coverage: + grids: [grid:de] + countries: [de, at] + entity_types: [Grid] + metrics: [reserve.capacity.mw, market.price.reserve, reserve.activation.mwh] + data_class: both + latency: tender results published per auction cycle (daily for FCR/aFRR/mFRR capacity) + resolution: 4h product slices (aFRR/mFRR), daily FCR + update_frequency: daily auction cycles + history_depth: multi-year tender archive + authentication: none + licensing: + class: open-attribution + notes: Public transparency publication of the German (and cooperating Austrian) balancing market; free CSV downloads. Activation/actual balancing data is on netztransparenz.de rather than here. + cost: free + reliability: 4 + quality_notes: Authoritative for German reserve procurement (demand, marginal/average capacity+energy prices). 50Hertz-specific grid data (wind/solar infeed, vertical load) is on 50hertz.com/Transparency and netztransparenz.de. + verification_status: documented_only + last_checked: "2026-08-12" + notes: CSV tender files per product/date; anonymised bid lists included. + + - source_id: source:epex:market-data + publisher: EPEX SPOT SE (EEX Group) + dataset: EPEX SPOT market data (day-ahead and intraday auction/continuous results, CWE + GB + Nordics) + api_type: rest + url: https://www.epexspot.com/en/marketdataservices + docs_url: https://www.epexspot.com/en/GCDataUse + coverage: + countries: [de, fr, at, be, nl, ch, gb, dk, fi, no, se, pl] + entity_types: [Grid, GridRegion] + metrics: [market.price.energy, market.volume.mwh] + data_class: both + latency: real-time/delayed/end-of-day tiers per subscription + resolution: hourly, 30min and 15min products + update_frequency: daily auction cycle; continuous intraday + history_depth: full exchange history via paid historical products + authentication: registration + licensing: + class: commercial + notes: COMMERCIAL - market data sold via EEX Group webshop (API/sFTP subscriptions invoiced annually); website day-ahead widgets are view-only under EPEX General Conditions and NOT licensed for systematic collection/redistribution. Day-ahead zone prices are available free via ENTSO-E TP instead. + cost: commercial + reliability: 4 + quality_notes: Primary exchange for continental European short-term power; only needed directly if intraday continuous depth (order book, VWAP) matters - otherwise ENTSO-E/SMARD suffice. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Recorded for completeness as the commercial upgrade path; no connector without a licence. + + - source_id: source:smard:market-data + publisher: Bundesnetzagentur (German Federal Network Agency) + dataset: SMARD - Strommarktdaten (German/European electricity market data platform) + api_type: rest + url: https://www.smard.de/en + docs_url: https://www.smard.de/en/datennutzung + coverage: + grids: [grid:de] + countries: [de] + entity_types: [Grid, GridRegion, Generator, Interconnector] + metrics: [grid.demand.mw, grid.generation.mw, market.price.energy, intercon.flow.mw] + data_class: both + latency: typically ~1-2h behind for actuals (mirrors ENTSO-E submission deadlines) + resolution: 15min (generation/consumption), hourly (prices) + update_frequency: continuous + history_depth: since 2015 + authentication: none + licensing: + class: open-attribution + notes: CC BY 4.0 - all published data free to download and reuse with credit to Bundesnetzagentur | SMARD.de. + cost: free + reliability: 4 + quality_notes: Regulator-curated German market view (generation by source, consumption, wholesale prices, cross-border flows, power plant data). Simple undocumented-but-stable JSON chart API plus a CSV download centre. + verification_status: documented_only + last_checked: "2026-08-12" + notes: JSON series at smard.de/app/chart_data/{filter}/{region}/..._{timestamp}.json (community-documented at github.com/bundesAPI/smard-api). diff --git a/intelligence/data/registries/sources/global-datasets.yaml b/intelligence/data/registries/sources/global-datasets.yaml new file mode 100644 index 0000000..1cd381d --- /dev/null +++ b/intelligence/data/registries/sources/global-datasets.yaml @@ -0,0 +1,313 @@ +# Source registry — Global reference datasets (structural, production/trade, satellite). See ../../schemas/source.schema.json +records: + - source_id: source:gem:global-coal-plant-tracker + publisher: Global Energy Monitor + dataset: Global Coal Plant Tracker (GCPT) - every coal-fired unit >=30MW worldwide + api_type: bulk-download + url: https://globalenergymonitor.org/projects/global-coal-plant-tracker/ + docs_url: https://globalenergymonitor.org/projects/global-coal-plant-tracker/download-data/ + coverage: + global: true + entity_types: [Generator, Company] + metrics: [asset.capacity.mw] + data_class: static + latency: n/a (structural reference) + resolution: unit level (capacity, status, fuel, owner, coordinates, GEM ID) + update_frequency: twice yearly (January and July releases) + history_depth: status history since 2010 (proposals) / 2000 (retirements) captured in fields + authentication: registration + licensing: + class: open-attribution + notes: CC BY 4.0; download requires filling a short form (name/email) - not an account. Cite as 'Global Coal Plant Tracker, Global Energy Monitor, [release]'. + cost: free + reliability: 4 + quality_notes: The de facto standard global coal fleet registry; GEM IDs map to entity external_ids.gem_id. Research-compiled - verify AU units against AEMO registration data where they overlap. + verification_status: documented_only + last_checked: "2026-08-12" + notes: XLSX download; latest release January 2026 at time of writing. + + - source_id: source:gem:global-coal-mine-tracker + publisher: Global Energy Monitor + dataset: Global Coal Mine Tracker (GCMT) - operating and proposed coal mines + api_type: bulk-download + url: https://globalenergymonitor.org/projects/global-coal-mine-tracker/ + docs_url: https://globalenergymonitor.org/projects/global-coal-mine-tracker/download-data/ + coverage: + global: true + entity_types: [Mine, Company] + metrics: [production.capacity.mtpa] + data_class: static + latency: n/a (structural reference) + resolution: mine level (capacity Mtpa, coal type, mine type, owner, coordinates, workforce) + update_frequency: roughly annual releases (with interim updates) + history_depth: current-state snapshot per release + authentication: registration + licensing: + class: open-attribution + notes: CC BY 4.0; form-based download like the plant tracker. + cost: free + reliability: 4 + quality_notes: Best open global mine registry (covers mines >=1Mtpa comprehensively, smaller where known); production capacities are estimates - calibrate AU mines against Queensland/NSW official statistics. + verification_status: documented_only + last_checked: "2026-08-12" + notes: XLSX; includes met vs thermal classification vital for the steel-chain model. + + - source_id: source:gem:global-gas-infrastructure-tracker + publisher: Global Energy Monitor + dataset: Global Gas Infrastructure Tracker (pipelines + LNG terminals; GGIT) + api_type: bulk-download + url: https://globalenergymonitor.org/projects/global-gas-infrastructure-tracker/ + docs_url: https://globalenergymonitor.org/projects/global-gas-infrastructure-tracker/download-data/ + coverage: + global: true + entity_types: [Pipeline, LngTerminal, Company] + metrics: [pipeline.capacity.bcm_y, lng.capacity.mtpa] + data_class: static + latency: n/a (structural reference) + resolution: pipeline/terminal level with routes (GeoJSON-style geometry for pipelines) + update_frequency: roughly twice yearly + history_depth: current-state snapshot per release + authentication: registration + licensing: + class: open-attribution + notes: CC BY 4.0; form-based download. + cost: free + reliability: 4 + quality_notes: Structural backbone for the gas/LNG chain (Curtis Island trains, east-coast pipelines); route geometries are approximate. + verification_status: documented_only + last_checked: "2026-08-12" + notes: XLSX + geometry files; complements AEMO GBB (live flows) with global static structure. + + - source_id: source:naturalearth:110m + publisher: Natural Earth (NACIS) + dataset: Natural Earth 1:110m cultural and physical vectors (countries, coastlines, land) + api_type: bulk-download + url: https://www.naturalearthdata.com/downloads/110m-cultural-vectors/ + docs_url: https://www.naturalearthdata.com/about/terms-of-use/ + coverage: + global: true + entity_types: [Country, Location] + metrics: [] + data_class: static + latency: n/a (basemap geometry) + resolution: 1:110,000,000 (globe-scale rendering) + update_frequency: occasional versioned releases (v5.x current line) + history_depth: versioned releases retained + authentication: none + licensing: + class: public-domain + notes: Public domain - free for any use, no attribution required (credit appreciated). Safe to COMMIT to the repo per LEGAL-LICENSING.md. + cost: free + reliability: 5 + quality_notes: Standard globe basemap; disputed-boundary variants available. Committed in this workspace for the deck.gl globe. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Shapefile/GeoJSON; 50m and 10m siblings if zoom detail is ever needed. + + - source_id: source:wikidata:sparql + publisher: Wikimedia Foundation / Wikidata community + dataset: Wikidata Query Service (SPARQL endpoint over the Wikidata knowledge graph) + api_type: rest + url: https://query.wikidata.org/sparql + docs_url: https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service + coverage: + global: true + entity_types: [Company, Generator, Port, Mine, Location] + metrics: [] + data_class: static + latency: live queries against a continuously edited graph (seconds-to-minutes edit lag) + resolution: entity level (QIDs) + update_frequency: continuous community edits + history_depth: full revision history per entity + authentication: none + licensing: + class: public-domain + notes: CC0 (public domain dedication) - maximally permissive. Query service etiquette - 60s query timeout, descriptive User-Agent, back off on 429s. + cost: free + reliability: 3 + quality_notes: Used for entity resolution/enrichment (QIDs as external_ids.wikidata_qid, links to LEI/GEM/OSM). Community-maintained - never a primary source for operational facts. + verification_status: documented_only + last_checked: "2026-08-12" + notes: SPARQL results as JSON/XML; linked-data fragments and dumps available for bulk work. + + - source_id: source:comtrade:api + publisher: United Nations Statistics Division + dataset: UN Comtrade - international merchandise trade statistics (HS commodity x partner) + api_type: rest + url: https://comtradeapi.un.org/ + docs_url: https://uncomtrade.org/docs/ + coverage: + global: true + entity_types: [Country, Commodity] + metrics: [trade.export.tonnes, trade.export.value.usd, trade.import.tonnes, trade.import.value.usd] + data_class: static + latency: monthly data lags ~2-3 months; annual consolidations later + resolution: reporter x partner x HS6 commodity, monthly/annual + update_frequency: continuous ingestion of national submissions; practical monthly cadence + history_depth: annual back to 1962; monthly for recent decades + authentication: api_key + licensing: + class: open-registration + notes: Free registration issues a subscription key - free tier ~500 calls/day, 100k records/call; premium tiers for bulk. Data free to reuse with attribution to UN Comtrade; bulk redistribution restricted. + cost: free tier; paid premium + reliability: 4 + quality_notes: The global ground truth for coal/LNG/alumina/aluminium trade flows at country level - calibrates inferred port-to-country flows. Mirror-flow asymmetries are a known feature; use reporter/partner reconciliation. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Key HS codes for this workspace - 2701 (coal), 2711 (gas/LNG), 2818 (alumina), 7601 (unwrought aluminium), 2601 (iron ore), 7207-72xx (steel). + + - source_id: source:jodi:world-databases + publisher: Joint Organisations Data Initiative (JODI) - IEA, OPEC, APEC, Eurostat, IEF, OLADE, UNSD, GECF + dataset: JODI-Oil and JODI-Gas World Databases (monthly national oil/gas production, demand, stocks, trade) + api_type: bulk-download + url: https://www.jodidata.org/ + docs_url: https://www.jodidata.org/oil/support/user-guide.aspx + coverage: + global: true + entity_types: [Country, Commodity] + metrics: [oil.production.kbd, gas.production.mcm, stocks.level.kt, trade.export.tonnes] + data_class: static + latency: ~1-2 months behind (data to one month old) + resolution: country x product x flow, monthly + update_frequency: monthly + history_depth: oil since January 2002; gas since January 2009 + authentication: none + licensing: + class: open-attribution + notes: Free downloads (CSV and Beyond-20/20 formats), no registration; cite JODI. Self-reported national data - coverage ~90 percent of markets with varying quality flags. + cost: free + reliability: 3 + quality_notes: The only free monthly global oil/gas balance; JODI-Gas covers LNG trade relevant to the Gladstone LNG exemplar. Quality codes per country-series must be honoured. + verification_status: documented_only + last_checked: "2026-08-12" + notes: One registry record covers both the oil and gas databases (same publisher/portal/terms). + + - source_id: source:usgs:mineral-commodity-summaries + publisher: U.S. Geological Survey (National Minerals Information Center) + dataset: Mineral Commodity Summaries (annual world production, reserves, trade by commodity) + api_type: bulk-download + url: https://www.usgs.gov/centers/national-minerals-information-center/mineral-commodity-summaries + docs_url: https://www.usgs.gov/centers/national-minerals-information-center + coverage: + global: true + entity_types: [Country, Commodity] + metrics: [production.rate.ktpa, reserves.tonnes] + data_class: static + latency: annual (published each January for the prior year, with estimates) + resolution: commodity x country, annual (~90 commodities) + update_frequency: annual (MCS); monthly/quarterly surveys for some commodities separately + history_depth: MCS editions back decades; historical statistics series to 1900 for many commodities + authentication: none + licensing: + class: public-domain + notes: US federal government work - public domain; data releases also on ScienceBase with DOIs and machine-readable CSV/XLSX. + cost: free + reliability: 4 + quality_notes: Reference series for bauxite/alumina/aluminium, iron ore and coal country-level production - anchors long-run capacity/production priors. + verification_status: documented_only + last_checked: "2026-08-12" + notes: 2026 edition covers 2025 estimates; per-commodity PDF + consolidated data files. + + - source_id: source:worldsteel:production-statistics + publisher: World Steel Association (worldsteel) + dataset: Monthly crude steel (+ iron/DRI) production statistics + api_type: html + url: https://worldsteel.org/data/ + docs_url: https://worldsteel.org/data/about-our-data/ + coverage: + global: true + entity_types: [Country, Commodity] + metrics: [production.crude_steel.mt, production.pig_iron.mt, production.dri.mt] + data_class: static + latency: monthly figures released ~22nd-25th of the following month + resolution: country level (~70 reporting countries, ~98 percent of world production), monthly + update_frequency: monthly press release + annual yearbook + history_depth: annual data back to 1900s via yearbooks; monthly series for recent years + authentication: none + licensing: + class: mixed + notes: Headline monthly press-release tables are free to cite with attribution; the detailed monthly subscription (~EUR 1590/yr) and Steel Statistical Yearbook data files are commercial. Record the split - free aggregates, paid detail. + cost: free headline; commercial detail + reliability: 4 + quality_notes: Industry-association self-reported but the accepted global benchmark - steel production drives met-coal demand, the demand side of the Bowen Basin chain. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Scrape-light approach - one press release per month; China (~half of world output) also cross-checkable against NBS. + + - source_id: source:iai:aluminium-statistics + publisher: International Aluminium Institute + dataset: IAI statistics - primary aluminium and alumina production (monthly, by region) + api_type: html + url: https://international-aluminium.org/statistics/primary-aluminium-production/ + docs_url: https://international-aluminium.org/statistics-overview/ + coverage: + global: true + entity_types: [Country, Commodity, Smelter] + metrics: [production.primary_aluminium.kt, production.alumina.kt] + data_class: static + latency: ~3 weeks after month end (e.g. monthly release around the 20th) + resolution: regional aggregates (incl. Oceania - covers AU smelters), monthly; annual by country detail + update_frequency: monthly + history_depth: primary aluminium series back to 1973; alumina to 1974 + authentication: none + licensing: + class: open-attribution + notes: Free to view/download from the IAI statistics pages with attribution; reproduction of substantial extracts needs IAI permission. Member-reported plus estimates of unreported production. + cost: free + reliability: 4 + quality_notes: The demand/production benchmark for the aluminium chain exemplar (Boyne Island smelter, Yarwun/QAL alumina) - Oceania regional series is the calibration line for inferred smelter utilisation. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Interactive charts with CSV export; regional not plant-level - plant attribution stays inferred. + + - source_id: source:copernicus:sentinel-2 + publisher: European Commission / ESA (Copernicus Data Space Ecosystem) + dataset: Sentinel-2 L1C/L2A optical imagery via CDSE (STAC/OData/openEO/Sentinel Hub APIs) + api_type: rest + url: https://dataspace.copernicus.eu/ + docs_url: https://documentation.dataspace.copernicus.eu/APIs.html + coverage: + global: true + entity_types: [Location, Facility, Port, Mine] + metrics: [satellite.image.scene] + data_class: both + latency: new acquisitions published within hours of downlink + resolution: 10-60m multispectral; ~5-day revisit (twin satellites) + update_frequency: continuous acquisition; per-tile revisit ~5 days + history_depth: full mission archive since 2015 + authentication: registration + licensing: + class: open-attribution + notes: Copernicus free, full and open data policy - free registration issues OAuth-style access tokens for download; generous free quotas, commercial terms only for very large-scale processing. Attribute Copernicus Sentinel data. + cost: free (quotas) + reliability: 4 + quality_notes: Visual/NDWI-style monitoring of stockpiles, berth occupancy, plant activity at named facilities - evidence imagery for inferences, cloud-cover dependent. + verification_status: documented_only + last_checked: "2026-08-12" + notes: STAC catalogue at stac.dataspace.copernicus.eu; also mirrored on AWS/Planetary Computer. Sentinel-1 SAR (all-weather ship/facility monitoring) available under the same ecosystem. + + - source_id: source:nasa:firms + publisher: NASA LANCE / University of Maryland + dataset: FIRMS - active fire / thermal anomaly detections (VIIRS 375m, MODIS 1km) + api_type: rest + url: https://firms.modaps.eosdis.nasa.gov/api/ + docs_url: https://firms.modaps.eosdis.nasa.gov/api/area/ + coverage: + global: true + entity_types: [Location, Facility, Smelter, Refinery] + metrics: [thermal.anomaly.frp_mw, thermal.anomaly.count] + data_class: live + latency: NRT within ~3h of overpass (ultra-real-time <1min for US/Canada) + resolution: 375m (VIIRS S-NPP/NOAA-20/NOAA-21), 1km (MODIS); ~2-4 overpasses/day/satellite + update_frequency: per satellite overpass + history_depth: archive back to 2000 (MODIS) / 2012 (VIIRS) via country/archive downloads + authentication: api_key + licensing: + class: open-registration + notes: Free MAP_KEY (instant email issue); 5000 transactions per 10-minute window. NASA data - effectively unrestricted with attribution; cite LANCE/FIRMS. + cost: free + reliability: 4 + quality_notes: NOTED USE - persistent industrial heat sources (smelters, steelworks, refinery flares, LNG flaring) appear as recurring thermal anomalies, so FIRMS doubles as a crude live activity/outage signal for facilities with no published telemetry - a documented mitigation for observability gaps. Confounded by bushfires - filter by persistence at fixed coordinates. + verification_status: documented_only + last_checked: "2026-08-12" + notes: CSV/JSON/WFS; area API takes bbox+date, e.g. poll a small bbox around Boyne Island / Curtis Island. diff --git a/intelligence/data/registries/sources/maritime.yaml b/intelligence/data/registries/sources/maritime.yaml new file mode 100644 index 0000000..183126c --- /dev/null +++ b/intelligence/data/registries/sources/maritime.yaml @@ -0,0 +1,261 @@ +# Source registry — Maritime & logistics. See ../../schemas/source.schema.json +records: + - source_id: source:aisstream:websocket + publisher: aisstream.io + dataset: AISStream real-time AIS websocket (terrestrial AIS network) + api_type: websocket + url: wss://stream.aisstream.io/v0/stream + docs_url: https://aisstream.io/documentation + coverage: + global: true + entity_types: [Vessel] + metrics: [vessel.position, vessel.speed.kn, vessel.course.deg, vessel.draught.m, vessel.destination] + data_class: live + latency: seconds (streaming) + resolution: per received AIS message; bounding-box + MMSI + message-type subscription filters + update_frequency: continuous stream + history_depth: none (live stream only - no replay/history) + authentication: api_key + licensing: + class: open-registration + notes: Free API key via sign-in (GitHub etc.); key sent inside the subscription JSON payload. Terms permit free programmatic use but NOT redistribution of the feed; terrestrial-only coverage (coastal), no satellite AIS. + cost: free + reliability: 3 + quality_notes: Community-grade terrestrial network - good coverage near major ports/coasts (fine for AU port approaches), open-ocean gaps by design. Design-target AIS source for this workspace per LEGAL-LICENSING.md. + verification_status: documented_only + last_checked: "2026-08-12" + notes: JSON messages (PositionReport, ShipStaticData etc.); reconnect logic required - single connection per key. + + - source_id: source:aishub:webservice + publisher: AISHub + dataset: AISHub aggregated AIS webservice (XML/JSON/CSV) + api_type: rest + url: https://www.aishub.net/api + docs_url: https://www.aishub.net/api + coverage: + global: true + entity_types: [Vessel] + metrics: [vessel.position, vessel.speed.kn, vessel.course.deg] + data_class: live + latency: near-real-time; polling not more than once per minute + resolution: latest position per vessel per poll + update_frequency: poll up to 1/min + history_depth: none (current positions only) + authentication: registration + licensing: + class: reciprocal + notes: PEER-EXCHANGE MODEL - free access requires CONTRIBUTING your own raw NMEA AIS feed (min ~10 vessels average coverage, 90 percent uptime over 7 days). No feed, no access. Data for members' own use. + cost: free for feed contributors + reliability: 3 + quality_notes: Terrestrial volunteer network. Relevant here as the documented upgrade path if Tailor deploys owned AIS receivers (e.g. Gladstone/Newcastle) - contributing one receiver unlocks the global aggregate. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Recorded as reciprocal-access, pending owned instrumentation; ties into the observability-gap registry for owned AIS receivers. + + - source_id: source:spire:maritime-ais + publisher: Spire Global + dataset: Spire Maritime - satellite + terrestrial + enhanced satellite AIS (API and TCP feeds) + api_type: rest + url: https://spire.com/maritime/ + docs_url: https://documentation.spire.com/ + coverage: + global: true + entity_types: [Vessel] + metrics: [vessel.position, vessel.speed.kn, vessel.course.deg, vessel.draught.m, vessel.destination] + data_class: both + latency: minutes (satellite revisit dependent) to seconds (terrestrial) + resolution: per AIS message; ~600k vessels detected daily + update_frequency: continuous + history_depth: historical AIS archive back to 2010 (paid) + authentication: api_key + licensing: + class: commercial + notes: COMMERCIAL subscription (GraphQL/REST API or raw TCP NMEA); no free tier. Documented as the production upgrade path for open-ocean coverage that terrestrial AIS cannot provide. + cost: commercial (subscription) + reliability: 4 + quality_notes: Own satellite constellation - genuine open-ocean coverage incl. Australia-Asia trade lanes; enhanced satellite AIS improves congested-area detection. + verification_status: documented_only + last_checked: "2026-08-12" + notes: No connector without a contract; recorded so cost/benefit of closing the open-ocean gap is explicit. + + - source_id: source:marinetraffic:api + publisher: MarineTraffic (a Kpler company) + dataset: MarineTraffic AIS data services (vessel positions, port calls, voyages) + api_type: rest + url: https://www.marinetraffic.com/ + docs_url: https://support.marinetraffic.com/en/articles/9552659-api-services + coverage: + global: true + entity_types: [Vessel, Port] + metrics: [vessel.position, vessel.speed.kn, vessel.draught.m, port.call.count, vessel.eta] + data_class: both + latency: near-real-time (terrestrial) / minutes (satellite) + resolution: per vessel position; port-call event granularity + update_frequency: continuous + history_depth: deep historical archive (paid) + authentication: api_key + licensing: + class: commercial + notes: COMMERCIAL ONLY - credit-based API retired in favour of enterprise subscriptions via Kpler sales. Website ToS PROHIBITS SCRAPING and any external dissemination or commercial use of displayed data without licence. Recorded explicitly so no connector is ever built against the public website. + cost: commercial (enterprise subscription) + reliability: 4 + quality_notes: Largest terrestrial network + satellite blend; strong port-call/ETA derived products. This workspace's licensing posture (LEGAL-LICENSING.md) rejects scraping it. + verification_status: documented_only + last_checked: "2026-08-12" + notes: UNCTAD port-call statistics are the licensed public derivative of MarineTraffic data - use that for aggregates. + + - source_id: source:kpler:platform + publisher: Kpler + dataset: Kpler commodity trade intelligence (cargo flows, tanker/dry-bulk tracking, inventories) + api_type: rest + url: https://www.kpler.com/ + docs_url: https://www.kpler.com/product/maritime + coverage: + global: true + entity_types: [Vessel, Port, Terminal, Commodity] + metrics: [cargo.flow.tonnes, vessel.position, port.call.count, inventory.level.tonnes] + data_class: both + latency: near-real-time AIS (~seconds-scale claimed); flow estimates updated daily + resolution: cargo/voyage level; terminal-level flows + update_frequency: continuous + history_depth: multi-year flow history + authentication: api_key + licensing: + class: commercial + notes: COMMERCIAL - enterprise SaaS + API, typically tens of thousands USD/yr upward. Cargo flows are Kpler's own INFERENCES from AIS + port + terminal data; if ingested they belong in the derived-intelligence class with Kpler as evidence, never as reported fact. + cost: commercial (enterprise) + reliability: 4 + quality_notes: Market-leading cargo-flow inference (owns MarineTraffic); brief record for the commercial upgrade path only. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Overlaps this workspace's own cargo-inference ambitions - useful benchmark, incompatible licence for redistribution. + + - source_id: source:vesselfinder:api + publisher: VesselFinder Ltd + dataset: VesselFinder AIS API (real-time positions, port calls, master data) + api_type: rest + url: https://api.vesselfinder.com/docs/ + docs_url: https://api.vesselfinder.com/docs/ + coverage: + global: true + entity_types: [Vessel, Port] + metrics: [vessel.position, vessel.speed.kn, vessel.draught.m, port.call.count] + data_class: both + latency: near-real-time + resolution: per vessel position (terrestrial 1 credit, satellite 10 credits) + update_frequency: continuous + history_depth: historical track/port-call products (paid) + authentication: api_key + licensing: + class: commercial + notes: COMMERCIAL credit-based API (from ~EUR 330 per 10k credits) - cheaper entry than enterprise-only competitors. Website ToS prohibits scraping; API is the only sanctioned programmatic path. + cost: commercial (pay-per-credit) + reliability: 3 + quality_notes: Credible mid-market alternative; brief record for the commercial upgrade path. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Credit pricing makes it viable for narrow queries (e.g. polling a handful of coal berths) if ever licensed. + + - source_id: source:unctad:port-statistics + publisher: UNCTAD (UN Trade and Development) + dataset: UNCTADstat port call and performance statistics (+ maritime transport indicators) + api_type: bulk-download + url: https://unctadstat.unctad.org/datacentre/reportInfo/US.PortCallsArrivals_S + docs_url: https://unctadstat.unctad.org/ + coverage: + global: true + entity_types: [Port, Country] + metrics: [port.call.count, port.time_in_port.days, vessel.size.gt.median] + data_class: static + latency: published semi-annually, ~months after period end + resolution: country x market segment (8 commercial vessel classes), semi-annual + update_frequency: twice yearly + history_depth: since 2018 (port calls); other maritime series longer + authentication: none + licensing: + class: open-attribution + notes: Free to use with attribution to UNCTAD; underlying data derived from MarineTraffic AIS under a licensed partnership (vessels >=1000 GT) - do not disaggregate beyond what UNCTAD publishes. + cost: free + reliability: 4 + quality_notes: The clean, licence-safe public aggregate of global port activity - baseline/benchmark data for port-level models, not live telemetry. + verification_status: documented_only + last_checked: "2026-08-12" + notes: CSV/Excel export from the UNCTADstat Data Centre; also mirrored on World Bank Data360. + + - source_id: source:unece:unlocode + publisher: UNECE (UN Centre for Trade Facilitation and Electronic Business) + dataset: UN/LOCODE - United Nations Code for Trade and Transport Locations + api_type: bulk-download + url: https://unece.org/trade/cefact/UNLOCODE-Download + docs_url: https://unece.org/trade/uncefact/unlocode + coverage: + global: true + entity_types: [Port, Location] + metrics: [] + data_class: static + latency: n/a (reference registry) + resolution: location level (~103k locations, 249 countries) + update_frequency: twice yearly (cut-offs 31 March / 30 September) + history_depth: all editions downloadable; codes are stable identifiers + authentication: none + licensing: + class: open-attribution + notes: Free UN publication, redistributable; also republished as a datahub.io/datasets GitHub data package under open terms. + cost: free + reliability: 5 + quality_notes: THE canonical port/location identifier scheme - used as external_ids.unlocode throughout this workspace's entity registry (e.g. AUGLT = Gladstone, AUNTL = Newcastle). + verification_status: documented_only + last_checked: "2026-08-12" + notes: CSV/MS-Access/text distributions; function classifiers (port, rail, airport) and coordinates included (coordinates are approximate). + + - source_id: source:gpc:cargo-statistics + publisher: Gladstone Ports Corporation (Queensland Government-owned) + dataset: GPC cargo statistics - monthly trade by commodity/terminal (Gladstone, Bundaberg, Rockhampton) + api_type: html + url: https://gpcl.com.au/ports-and-trade/ + docs_url: https://gpcl.com.au/ + coverage: + countries: [au] + entity_types: [Port, Terminal, Commodity] + metrics: [export.tonnes.mt, import.tonnes.mt, port.throughput.tonnes, vessel.call.count] + data_class: static + latency: ~weeks after month end + resolution: monthly, by commodity (coal, LNG, alumina, bauxite, cement etc.) and trade direction + update_frequency: monthly + history_depth: multi-year published statistics; annual reports deeper + authentication: none + licensing: + class: open-attribution + notes: Published on GPC's public website; Queensland GOC publication - cite GPC. No machine API; figures also flow into TMR's 'Trade statistics for Queensland ports' annual compendium. + cost: free + reliability: 4 + quality_notes: Ground truth for the Gladstone worked exemplar - RG Tanna/Barney Point coal, Curtis Island LNG, alumina chain tonnages. Monthly cadence makes it the calibration series for cargo inference from AIS. + verification_status: documented_only + last_checked: "2026-08-12" + notes: HTML tables/downloadable reports; historically a 'Trade Statistics' interactive page - scrape gently and cache. + + - source_id: source:pon:trade-reports + publisher: Port of Newcastle + dataset: Port of Newcastle monthly trade reports and vessel statistics + api_type: html + url: https://pon.com.au/trade-and-business/trade-overview-reports/ + docs_url: https://pon.com.au/ + coverage: + countries: [au] + entity_types: [Port, Commodity] + metrics: [export.tonnes.mt, import.tonnes.mt, vessel.call.count, port.throughput.tonnes] + data_class: static + latency: ~weeks after month end + resolution: monthly + YTD, by commodity group (coal ~149Mt/yr dominant) + update_frequency: monthly + history_depth: monthly reports back to January 2018 + authentication: none + licensing: + class: open-attribution + notes: Public reports on the port's website; cite Port of Newcastle. NSW Transport also republishes port freight data. + cost: free + reliability: 4 + quality_notes: The world's largest coal export port - monthly tonnage ground truth for the Hunter Valley chain (pairs with ARTC/Aurizon rail context and AIS vessel observation). + verification_status: documented_only + last_checked: "2026-08-12" + notes: PDF/interactive monthly trade reports; daily shipping movements (expected/at-berth) also published on the site's shipping pages - live-ish but scrape-sensitive. diff --git a/intelligence/data/registries/sources/north-america.yaml b/intelligence/data/registries/sources/north-america.yaml new file mode 100644 index 0000000..fc08da4 --- /dev/null +++ b/intelligence/data/registries/sources/north-america.yaml @@ -0,0 +1,296 @@ +# Source registry — North America. See ../../schemas/source.schema.json +records: + - source_id: source:eia:api-v2 + publisher: U.S. Energy Information Administration + dataset: EIA Open Data API v2 (electricity, natural gas, petroleum, coal series) + api_type: rest + url: https://api.eia.gov/v2/ + docs_url: https://www.eia.gov/opendata/documentation.php + coverage: + countries: [us] + entity_types: [Grid, GridRegion, Generator, Mine, Pipeline] + metrics: [grid.demand.mw, grid.generation.mw, power.output.mw, production.rate.tph, storage.level.pct] + data_class: both + latency: hourly series ~1-2h behind; monthly/annual series lag weeks to months + resolution: hourly (grid monitor) to annual, per series + update_frequency: varies by route - hourly for electricity/rto, monthly/annual for most statistical series + history_depth: decades for statistical series; hourly grid data since July 2015 + authentication: api_key + licensing: + class: public-domain + notes: US federal government work - public domain; free API key required (instant issue), rate limits apply (~5000 req/hr). + cost: free + reliability: 5 + quality_notes: Authoritative US statistical agency; v2 API is a facet/route model (JSON). Plant-level data via electricity routes joins on EIA plant code. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Bulk download files also available for full-dataset sync without hammering the API. + + - source_id: source:eia:eia930 + publisher: U.S. Energy Information Administration + dataset: EIA-930 Hourly Electric Grid Monitor (balancing authority demand, generation by fuel, interchange) + api_type: rest + url: https://api.eia.gov/v2/electricity/rto/ + docs_url: https://www.eia.gov/electricity/gridmonitor/about + coverage: + countries: [us] + entity_types: [Grid, GridRegion, Interconnector] + metrics: [grid.demand.mw, grid.generation.mw, intercon.flow.mw, grid.demand.forecast.mw] + data_class: live + latency: ~1-2 hours behind real time + resolution: hourly, per balancing authority (~65 BAs) + update_frequency: hourly + history_depth: since July 2015; fuel-mix detail since July 2018 + authentication: api_key + licensing: + class: public-domain + notes: US federal government work - public domain; same free EIA API key as api-v2. + cost: free + reliability: 4 + quality_notes: The only consistent cross-US BA-level hourly picture (demand, net generation by fuel, interchange); BA-reported with known reporting gaps/imputation - EIA flags imputed values. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Also downloadable as bulk CSV from the Grid Monitor UI; sub-BA demand available for larger BAs. + + - source_id: source:caiso:oasis + publisher: California Independent System Operator + dataset: CAISO OASIS (Open Access Same-time Information System) + api_type: rest + url: http://oasis.caiso.com/oasisapi/SingleZip + docs_url: https://www.caiso.com/library/oasis-frequently-asked-questions + coverage: + grids: [grid:us-caiso] + countries: [us] + entity_types: [Grid, GridRegion, Interconnector] + metrics: [market.price.energy, grid.demand.mw, grid.generation.mw, intercon.flow.mw] + data_class: both + latency: minutes for real-time market results + resolution: 5min (RTM/RTD), 15min (FMM), hourly (DAM) + update_frequency: per market run + history_depth: years of history queryable by date range + authentication: none + licensing: + class: open-attribution + notes: Public market transparency data; no key required. CAISO terms permit use of OASIS data; heavy automated use should respect published request limits. + cost: free + reliability: 4 + quality_notes: LMPs at thousands of nodes, system demand/forecast, fuel mix, interchange; quirky legacy API (ZIP-of-CSV/XML responses, strict query windows) but authoritative. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Today's Outlook dashboard JSON endpoints are a simpler live alternative for headline fuel-mix/demand. + + - source_id: source:pjm:dataminer2 + publisher: PJM Interconnection + dataset: PJM Data Miner 2 API (market and operations data) + api_type: rest + url: https://api.pjm.com/api/v1/ + docs_url: https://dataminer2.pjm.com/ + coverage: + grids: [grid:us-pjm] + countries: [us] + entity_types: [Grid, GridRegion, Interconnector] + metrics: [market.price.energy, grid.demand.mw, grid.generation.mw, intercon.flow.mw] + data_class: both + latency: minutes for instantaneous load/LMP feeds + resolution: 5min to hourly depending on feed + update_frequency: per feed - 5min for real-time LMP/load, hourly/daily for others + history_depth: multi-year history per feed (varies) + authentication: api_key + licensing: + class: open-registration + notes: Free but requires a PJM Tools account and a subscription key (Ocp-Apim-Subscription-Key header); non-members request access via DataMiner2Support@pjm.com. Data provided for informational purposes under PJM terms. + cost: free + reliability: 4 + quality_notes: Comprehensive REST/JSON coverage of the largest US RTO (LMPs, load, generation by fuel, interchange, ancillary); the web UI needs no login, API does. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Feed catalogue and per-feed schema browsable at dataminer2.pjm.com. + + - source_id: source:ercot:mis + publisher: Electric Reliability Council of Texas + dataset: ERCOT public reports (MIS) and Public API + api_type: rest + url: https://api.ercot.com/ + docs_url: https://developer.ercot.com/ + coverage: + grids: [grid:us-ercot] + countries: [us] + entity_types: [Grid, GridRegion] + metrics: [market.price.energy, grid.demand.mw, grid.generation.mw, grid.frequency.hz] + data_class: both + latency: minutes for real-time dashboards and SCED results + resolution: 5min (SCED/real-time), 15min settlement, hourly DAM + update_frequency: per report - many at 5min cadence + history_depth: years via the Data Access Portal archive + authentication: api_key + licensing: + class: open-registration + notes: Public reports are free; the Public API (api.ercot.com) requires free registration for a client ID + subscription key under ERCOT API Terms of Use. Certificate-secured MIS content is participant-only. + cost: free + reliability: 4 + quality_notes: Real-time prices, load, fuel mix, wind/solar production and forecasts; the modern Public API/Data Access Portal (data.ercot.com) supersedes scraping the legacy MIS report site. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Legacy public report listings remain at www.ercot.com/mp/data-products; grid conditions dashboard offers simple JSON. + + - source_id: source:miso:market-data + publisher: Midcontinent Independent System Operator + dataset: MISO market reports, real-time web display APIs, and Data Exchange + api_type: rest + url: https://api.misoenergy.org/MISORTWDDataBroker/data-links.html + docs_url: https://www.misoenergy.org/markets-and-operations/rtdataapis/ + coverage: + grids: [grid:us-miso] + countries: [us, ca] + entity_types: [Grid, GridRegion, Interconnector] + metrics: [market.price.energy, grid.demand.mw, grid.generation.mw, intercon.flow.mw] + data_class: both + latency: minutes for real-time data broker feeds + resolution: 5min real-time; hourly DA/RT ex-post + update_frequency: 5min for RT web display feeds; daily for market reports + history_depth: market report archive spans years; RT feeds are snapshot-only + authentication: none + licensing: + class: open-registration + notes: RT web-display JSON feeds are open; MISO is migrating market reports to the registration-based (free) MISO Data Exchange API - plan for account creation. MISO terms permit informational use. + cost: free + reliability: 4 + quality_notes: Real-time feeds are JSON-only as of Dec 2025. Legacy CSV/XLS market reports being phased out in favour of Data Exchange. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Covers 15 US states + Manitoba - hence countries [us, ca]. + + - source_id: source:ieso:public-reports + publisher: Independent Electricity System Operator (Ontario) + dataset: IESO public reports directory (XML/CSV market and operations reports) + api_type: file-drop + url: https://reports-public.ieso.ca/public/ + docs_url: https://www.ieso.ca/power-data/data-directory + coverage: + grids: [grid:ca-ieso] + countries: [ca] + entity_types: [Grid, Generator, Interconnector] + metrics: [grid.demand.mw, power.output.mw, market.price.energy, intercon.flow.mw] + data_class: both + latency: minutes (5-min reports) to hourly + resolution: 5min and hourly; generator-level hourly output/capability + update_frequency: per report - 5min, hourly, daily + history_depth: multi-year archive in the same directory tree + authentication: none + licensing: + class: open-attribution + notes: Public reports free to use under IESO terms of use with attribution. + cost: free + reliability: 4 + quality_notes: Unusually rich open unit-level data (GenOutputCapability XML per generator, hourly). Ontario's May 2025 market renewal moved pricing from HOEP to LMP - report set changed; use current data directory. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Plain directory listing of versioned XML/CSV files - trivially pollable. + + - source_id: source:nyiso:oasis + publisher: New York Independent System Operator + dataset: NYISO OASIS / public CSV market data (mis.nyiso.com) + api_type: file-drop + url: http://mis.nyiso.com/public/ + docs_url: https://www.nyiso.com/energy-market-operational-data + coverage: + grids: [grid:us-nyiso] + countries: [us] + entity_types: [Grid, GridRegion, Interconnector] + metrics: [market.price.energy, grid.demand.mw, intercon.flow.mw, grid.generation.mw] + data_class: both + latency: ~5min for real-time LBMP and load + resolution: 5min (RT), hourly (DA); zonal and generator-node LBMPs + update_frequency: every 5 minutes for RT files; daily rollups + history_depth: daily/monthly CSV archives back to ~1999 + authentication: none + licensing: + class: open-attribution + notes: Public market data; free with attribution under NYISO terms. + cost: free + reliability: 4 + quality_notes: Predictable CSV URL patterns (e.g. /public/csv/realtime/{yyyymmdd}realtime_zone.csv) make this one of the easiest ISOs to ingest without an API. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Fuel mix, interface flows, load forecast files under parallel csv directories. + + - source_id: source:isone:web-services + publisher: ISO New England + dataset: ISO-NE Web Services API v1.1 (ISO Express data) + api_type: rest + url: https://webservices.iso-ne.com/api/v1.1 + docs_url: https://webservices.iso-ne.com/docs/v1.1/ + coverage: + grids: [grid:us-isone] + countries: [us] + entity_types: [Grid, GridRegion, Interconnector] + metrics: [market.price.energy, grid.demand.mw, grid.generation.mw, intercon.flow.mw] + data_class: both + latency: minutes for real-time series + resolution: 5min (RT LMP, load) and hourly + update_frequency: per endpoint - 5min to daily + history_depth: multi-year via date-parameterised endpoints + authentication: registration + licensing: + class: open-registration + notes: Free ISO Express account grants API access via HTTP Basic auth over TLS; ISO-NE data terms permit informational use with attribution. + cost: free + reliability: 4 + quality_notes: Clean REST (XML/JSON) over all public ISO Express content - fuel mix, LMPs, demand, reserves. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Sandbox at ws-sandbox.iso-ne.com for development. + + - source_id: source:aeso:ets + publisher: Alberta Electric System Operator + dataset: AESO Energy Trading System (ETS) public reports and AESO API (pool price, AIL, generation) + api_type: rest + url: https://api.aeso.ca/ + docs_url: https://ets.aeso.ca/ + coverage: + grids: [grid:ca-aeso] + countries: [ca] + entity_types: [Grid, Generator, Interconnector] + metrics: [market.price.energy, grid.demand.mw, power.output.mw, intercon.flow.mw] + data_class: both + latency: minutes (current supply/demand report is near-real-time) + resolution: hourly pool price / Alberta Internal Load; asset-level MW in current supply-demand report + update_frequency: continuous for CSD report; hourly for price series + history_depth: ETS historical reports back to 2000 + authentication: api_key + licensing: + class: open-registration + notes: ETS web reports are open; the AESO API portal (api.aeso.ca, e.g. Pool Price and Current Supply Demand APIs) issues free keys on registration. AESO terms permit informational use. + cost: free + reliability: 4 + quality_notes: Alberta is an energy-only market with unit-level current MW/max capability in the CSD report - one of the few live asset-level Canadian feeds. Restructured Energy Market changes (2026-27) may alter report formats. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Legacy CSD HTML at ets.aeso.ca is scrape-fragile; prefer the JSON API. + + - source_id: source:hydroquebec:open-data + publisher: Hydro-Québec + dataset: Hydro-Québec open data portal (electricity demand, generation/consumption history) + api_type: rest + url: https://donnees.hydroquebec.com/ + docs_url: https://www.hydroquebec.com/documents-data/open-data/ + coverage: + grids: [grid:ca-hq] + countries: [ca] + entity_types: [Grid] + metrics: [grid.demand.mw, grid.generation.mw] + data_class: both + latency: near-real-time for the demand dataset + resolution: 15min (Québec electricity demand); hourly proxies for generation/consumption history + update_frequency: demand dataset refreshed every 15 minutes + history_depth: multi-year history datasets published separately + authentication: none + licensing: + class: open-noncommercial + notes: CC BY-NC 4.0 - attribution required and COMMERCIAL USE PROHIBITED without separate arrangement. Flag before any commercial product use. + cost: free + reliability: 4 + quality_notes: Opendatasoft platform with clean REST/records API; covers Québec except autonomous networks. The NC licence is the key constraint, not data quality. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Key dataset - demande-electricite-quebec (province-wide demand in MW). diff --git a/intelligence/data/registries/sources/weather.yaml b/intelligence/data/registries/sources/weather.yaml new file mode 100644 index 0000000..c0e5288 --- /dev/null +++ b/intelligence/data/registries/sources/weather.yaml @@ -0,0 +1,105 @@ +# Source registry — Weather & environment. See ../../schemas/source.schema.json +records: + - source_id: source:bom:observations + publisher: Australian Bureau of Meteorology + dataset: BOM station observations (JSON/XML data feeds) and warnings + api_type: file-drop + url: http://www.bom.gov.au/catalogue/data-feeds.shtml + docs_url: http://www.bom.gov.au/catalogue/data-feeds.shtml + coverage: + countries: [au] + entity_types: [WeatherStation] + metrics: [weather.temp.c, weather.wind.speed_ms, weather.wind.dir.deg, weather.pressure.hpa, weather.rain.mm, weather.humidity.pct] + data_class: live + latency: ~10-30min after observation time + resolution: station-level; observations typically half-hourly (10-min at some stations) + update_frequency: every 10-30 minutes per station product + history_depth: rolling 72h in the JSON/axf feeds; long climate archives via separate climate data services + authentication: none + licensing: + class: open-attribution + notes: BOM copyright permits use with attribution for most published products; feeds are free and 'not for commercial use' in the anonymous channel - a registered/paid channel exists for high-volume/commercial programmatic access. BOM's WAF blocks HTML screen-scraping; the XML/JSON/FTP data feeds are the sanctioned path. + cost: free (registered/paid channel for high-volume commercial use) + reliability: 4 + quality_notes: Authoritative AU observations; product IDs per state/product (e.g. IDQ60801 QLD observations). Low-cadence station polling by design per this workspace's licensing posture. + verification_status: documented_only + last_checked: "2026-08-12" + notes: JSON per-station at www.bom.gov.au/fwo/{product}/{product}.{wmo-id}.json; bulk via ftp.bom.gov.au/anon/gen/fwo/. Set a proper User-Agent and modest cadence. + + - source_id: source:noaa:nws-api + publisher: US National Weather Service (NOAA) + dataset: NWS API (api.weather.gov) - forecasts, observations, alerts + api_type: rest + url: https://api.weather.gov/ + docs_url: https://www.weather.gov/documentation/services-web-api + coverage: + countries: [us] + entity_types: [WeatherStation] + metrics: [weather.temp.c, weather.wind.speed_ms, weather.pressure.hpa, weather.precip.mm] + data_class: live + latency: minutes for station observations and alerts + resolution: station observations (typically hourly METAR-based); 2.5km gridded forecasts + update_frequency: hourly observations; forecasts per NWS cycle; alerts near-instant + history_depth: recent observations only via API; deep archives via NCEI + authentication: none + licensing: + class: public-domain + notes: US government work - public domain. No API key; a descriptive User-Agent header is required and rate limits are enforced generously. + cost: free + reliability: 5 + quality_notes: Clean JSON-LD REST API; relevant for US grid-region weather context (ERCOT/CAISO/PJM demand drivers). + verification_status: documented_only + last_checked: "2026-08-12" + notes: points/{lat},{lon} resolves to gridpoints and stations; alerts endpoint useful for extreme-weather event flags. + + - source_id: source:openmeteo:forecast-api + publisher: Open-Meteo + dataset: Open-Meteo forecast and historical weather API (multi-model) + api_type: rest + url: https://api.open-meteo.com/v1/forecast + docs_url: https://open-meteo.com/en/docs + coverage: + global: true + entity_types: [Location] + metrics: [weather.temp.c, weather.wind.speed_ms, weather.solar.ghi.wm2, weather.precip.mm] + data_class: both + latency: models refreshed on their native cycles (hourly to 6-hourly) + resolution: hourly (15-min for some variables/regions); point queries by lat/lon + update_frequency: per underlying model run (ICON, GFS, ECMWF IFS/AIFS, ACCESS-G, etc.) + history_depth: historical weather API back to 1940 (ERA5); forecast archive since 2022 + authentication: none + licensing: + class: open-noncommercial + notes: Free WITHOUT key for non-commercial use (fair-use ~10k calls/day); commercial use requires a paid API subscription. Data/API content under CC BY 4.0 attribution; underlying model data from open national providers. + cost: free non-commercial; paid commercial tiers + reliability: 4 + quality_notes: Excellent developer ergonomics (single JSON call, model blending) - ideal for wind/solar generation context at plant coordinates. Not an observation network - model output. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Dedicated endpoints for ECMWF, historical (archive-api), air quality, marine; solar radiation variables suit PV output modelling. + + - source_id: source:ecmwf:open-data + publisher: European Centre for Medium-Range Weather Forecasts + dataset: ECMWF Open Data - real-time IFS and AIFS forecast products (0.25 deg) + api_type: bulk-download + url: https://data.ecmwf.int/forecasts/ + docs_url: https://confluence.ecmwf.int/display/DAC/ECMWF+open+data%3A+real-time+forecasts+from+IFS+and+AIFS + coverage: + global: true + entity_types: [Location] + metrics: [weather.temp.c, weather.wind.speed_ms, weather.pressure.hpa, weather.precip.mm] + data_class: live + latency: published on the real-time dissemination schedule (~hours after synoptic time) + resolution: 0.25 degree grid, GRIB2; 3-6 hourly steps to 10+ days + update_frequency: 4 runs/day (00/06/12/18Z; AIFS as available) + history_depth: rolling recent runs on the open endpoint; archives via MARS/CDS + authentication: none + licensing: + class: open-attribution + notes: LICENCE CHANGE - formerly restricted, ECMWF real-time products moved to open data under CC BY 4.0 (resolution upgraded to 0.25 deg in 2024; ECMWF has since opened its entire real-time catalogue). Attribute ECMWF. + cost: free + reliability: 5 + quality_notes: World-reference global NWP now genuinely open, including the AIFS ML model - the global forecast backbone for demand/renewables context outside AU/US national services. + verification_status: documented_only + last_checked: "2026-08-12" + notes: Also mirrored on AWS Open Data (registry.opendata.aws/ecmwf-forecasts); python client 'ecmwf-opendata' simplifies retrieval. diff --git a/intelligence/schemas/source.schema.json b/intelligence/schemas/source.schema.json index 88d6f58..27689b7 100644 --- a/intelligence/schemas/source.schema.json +++ b/intelligence/schemas/source.schema.json @@ -4,68 +4,156 @@ "title": "Source", "description": "A registered data source (publisher + dataset) in the source registry.", "type": "object", - "required": ["source_id", "publisher", "dataset", "api_type", "url", "coverage", "data_class", "licensing", "verification_status", "last_checked"], + "required": [ + "source_id", + "publisher", + "dataset", + "api_type", + "url", + "coverage", + "data_class", + "licensing", + "verification_status", + "last_checked" + ], "properties": { - "source_id": { "$ref": "defs.schema.json#/$defs/sourceId" }, - "publisher": { "type": "string" }, - "dataset": { "type": "string" }, + "source_id": { + "$ref": "defs.schema.json#/$defs/sourceId" + }, + "publisher": { + "type": "string" + }, + "dataset": { + "type": "string" + }, "api_type": { - "enum": ["rest", "websocket", "file-drop", "html", "bulk-download", "none"] + "enum": [ + "rest", + "websocket", + "file-drop", + "html", + "bulk-download", + "none" + ] + }, + "url": { + "type": "string" + }, + "docs_url": { + "type": "string" }, - "url": { "type": "string" }, - "docs_url": { "type": "string" }, "coverage": { "type": "object", - "required": ["grids", "countries"], "properties": { "grids": { "type": "array", - "items": { "type": "string" } + "items": { + "type": "string" + } }, "countries": { "type": "array", - "items": { "type": "string" } + "items": { + "type": "string" + } + }, + "global": { + "type": "boolean" } }, "additionalProperties": false }, "data_class": { - "enum": ["live", "static", "both"] + "enum": [ + "live", + "static", + "both" + ] }, "entity_types": { "type": "array", - "items": { "type": "string" } + "items": { + "type": "string" + } }, "metrics": { "type": "array", - "items": { "type": "string" } + "items": { + "type": "string" + } + }, + "latency": { + "type": "string" + }, + "resolution": { + "type": "string" + }, + "update_frequency": { + "type": "string" + }, + "history_depth": { + "type": "string" }, - "latency": { "type": "string" }, - "resolution": { "type": "string" }, - "update_frequency": { "type": "string" }, - "history_depth": { "type": "string" }, "licensing": { "type": "object", - "required": ["class"], + "required": [ + "class" + ], "properties": { "class": { - "enum": ["open", "open-attribution", "registration-required", "commercial", "restricted"] + "enum": [ + "open", + "public-domain", + "open-attribution", + "open-registration", + "open-noncommercial", + "open-sharealike", + "reciprocal", + "registration-required", + "commercial", + "restricted", + "mixed" + ] }, - "notes": { "type": "string" } + "notes": { + "type": "string" + } }, "additionalProperties": false }, "authentication": { - "enum": ["none", "api_key", "oauth", "registration"] + "enum": [ + "none", + "api_key", + "oauth", + "registration" + ] + }, + "cost": { + "type": "string" + }, + "reliability": { + "type": "integer", + "minimum": 1, + "maximum": 5 }, - "cost": { "type": "string" }, - "reliability": { "type": "integer", "minimum": 1, "maximum": 5 }, "verification_status": { - "enum": ["verified_live", "documented_only", "unverified"] + "enum": [ + "verified_live", + "documented_only", + "unverified" + ] + }, + "last_checked": { + "type": "string", + "format": "date" + }, + "quality_notes": { + "type": "string" }, - "last_checked": { "type": "string", "format": "date" }, - "quality_notes": { "type": "string" }, - "notes": { "type": "string" } + "notes": { + "type": "string" + } }, "additionalProperties": false -} +} \ No newline at end of file From 722541edc607b1d33d4a43c7591fb990ca449dc4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 01:52:40 +0000 Subject: [PATCH 10/16] =?UTF-8?q?feat(intelligence):=20observability=20gap?= =?UTF-8?q?=20registry=20=E2=80=94=2035=20NEM/WEM=20gaps=20(phase=2010)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Where live observability does not exist, the gap is explicit and first-class: Boyne potline load, RG Tanna/WICET shiploader throughput, per-train LNG production at Curtis Island, Bowen Basin mine production, coal-rail consist movements, Tomago, Port Kembla, Sydney data-centre load, NWIS (no public real-time data at all). Each records the best available proxy with its derivation and the owned-instrumentation options (substation gateways, AIS receivers, partner feeds, satellite thermal) that could close it. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014yVGqBtvM1zNAd9tvUVNzm --- .../registries/gaps/au-nem-gladstone.yaml | 290 +++++++++++++++ .../data/registries/gaps/au-nem-other.yaml | 350 ++++++++++++++++++ .../data/registries/gaps/au-wem-nwis.yaml | 80 ++++ 3 files changed, 720 insertions(+) create mode 100644 intelligence/data/registries/gaps/au-nem-gladstone.yaml create mode 100644 intelligence/data/registries/gaps/au-nem-other.yaml create mode 100644 intelligence/data/registries/gaps/au-wem-nwis.yaml diff --git a/intelligence/data/registries/gaps/au-nem-gladstone.yaml b/intelligence/data/registries/gaps/au-nem-gladstone.yaml new file mode 100644 index 0000000..74f0a27 --- /dev/null +++ b/intelligence/data/registries/gaps/au-nem-gladstone.yaml @@ -0,0 +1,290 @@ +# Observability-gap registry — Gladstone industrial cluster (NEM / QLD1). See ../../schemas/observability-gap.schema.json +records: + - gap_id: gap:au-nem:boyne-live-load + entity_id: smelter:au-qld:boyne-island + desired_metric: power.load.mw + current_source: null + best_available_proxy: + source_id: source:aemo:nemweb-dispatch-scada + metric: grid.demand.mw + derivation: QLD1 regional demand net of scheduled generation; smelter is ~10% of QLD load and its potline load is near-constant, so residual-load deltas at known interruption events isolate it + required_resolution: 1min + required_accuracy: ±1% + commercial_value: high + strategic_value: critical + instrumentation_options: + - { kind: substation_gateway, notes: Boyne smelter is fed by dedicated 275 kV circuits from Calvale/Gladstone; a gateway at the smelter substation would give exact potline load, indicative_cost_band: high } + - { kind: revenue_meter, notes: integration with the retailer/TNSP revenue meter via customer API, indicative_cost_band: medium } + - { kind: customer_api, notes: direct data-sharing agreement with Rio Tinto, indicative_cost_band: low } + estimated_complexity: medium + priority: P1 + status: open + notes: Largest single electricity consumer in Queensland; ~950-1000 MW continuous draw, ~8 TWh/yr. Its state is the keystone of Gladstone-cluster intelligence. + + - gap_id: gap:au-nem:boyne-production-rate + entity_id: smelter:au-qld:boyne-island + desired_metric: production.rate.tph + current_source: null + best_available_proxy: + source_id: source:aemo:nemweb-dispatch-scada + metric: grid.demand.mw + derivation: "Potline load divided by specific energy of 13.5-14 MWh/t gives hot-metal rate; at ~950 MW that is ~68-70 t/h, ~500-545 kt/yr against a 585 kt/yr nameplate (three potlines, one partially curtailed since 2017). Load-shedding events longer than ~3 h imply pot instability and multi-week production loss." + required_resolution: 1h + required_accuracy: ±5% + commercial_value: high + strategic_value: high + instrumentation_options: + - { kind: customer_api, notes: "Rio Tinto production data-sharing agreement; smelter output is metered internally per potline", indicative_cost_band: low } + - { kind: substation_gateway, notes: "exact potline MW from an owned substation gateway makes the MWh/t conversion accurate to ~2%", indicative_cost_band: high } + - { kind: partner_feed, notes: "GPC aluminium/alumina wharf tonnage feed reconciles inferred production monthly", indicative_cost_band: medium } + estimated_complexity: medium + priority: P2 + status: open + notes: Production is an almost deterministic function of potline load, so this gap closes automatically once gap:au-nem:boyne-live-load is instrumented. + + - gap_id: gap:au-nem:qal-alumina-production + entity_id: refinery:au-qld:qal + desired_metric: production.rate.tph + current_source: null + best_available_proxy: + source_id: source:aisstream:websocket + metric: vessel.movement.events + derivation: "QAL (~3.7 Mtpa, ~420-460 t/h) runs a closed Weipa-Gladstone bauxite shuttle at ~2.3 t bauxite per t alumina; counting bauxite-carrier arrivals at the QAL wharf (parcels ~70-90 kt) and alumina/caustic vessel calls gives a ~weekly production nowcast, reconciled monthly against GPC cargo statistics." + required_resolution: 1d + required_accuracy: ±10% + commercial_value: high + strategic_value: high + instrumentation_options: + - { kind: customer_api, notes: "data-sharing agreement with Rio Tinto (80% owner and operator; Rusal's 20% is sanctions-affected and Rio has sole operational control since 2022)", indicative_cost_band: low } + - { kind: camera_ocr, notes: "camera on the QAL wharf unloader/shiploader with OCR of vessel names plus belt-motion detection", indicative_cost_band: medium } + - { kind: partner_feed, notes: "GPC per-wharf tonnage feed at daily rather than monthly cadence", indicative_cost_band: medium } + estimated_complexity: medium + priority: P2 + status: open + notes: One of the world's largest alumina refineries, operating since 1967; feeds Boyne and export markets. Bauxite inventory on site (~2-3 weeks) smooths the AIS proxy. + + - gap_id: gap:au-nem:qal-gas-intake + entity_id: refinery:au-qld:qal + desired_metric: gas.consumption.tj + current_source: null + best_available_proxy: + source_id: source:aemo:gas-bulletin-board + metric: pipeline.deliveries.tj + derivation: "Queensland Gas Pipeline (Wallumbilla-Gladstone) daily deliveries into the Gladstone demand zone, published D+1, less the Yarwun cogeneration/calcination baseline and the Orica baseline, leaves the QAL steam/calcination gas share; step-changes greater than ~10 TJ/d flag digester or calciner outages." + required_resolution: 1h + required_accuracy: ±5% + commercial_value: medium + strategic_value: high + instrumentation_options: + - { kind: customer_api, notes: "Rio Tinto energy-desk data share covering QAL fuel intake", indicative_cost_band: low } + - { kind: partner_feed, notes: "pipeline-operator (Jemena QGP) delivery-point feed for the Gladstone laterals", indicative_cost_band: medium } + - { kind: iot_gateway, notes: "owned gateway on the site gas metering skid under a service agreement", indicative_cost_band: high } + estimated_complexity: medium + priority: P3 + status: open + notes: Gas intake is the fastest-moving observable of refinery run-rate and of Gladstone's competing claim on east-coast gas; GBB zone granularity hides the per-customer split. + + - gap_id: gap:au-nem:yarwun-production + entity_id: refinery:au-qld:yarwun + desired_metric: production.rate.tph + current_source: null + best_available_proxy: + source_id: source:aemo:gas-bulletin-board + metric: pipeline.deliveries.tj + derivation: "Yarwun (~3 Mtpa, ~340 t/h) is gas-fired (on-site cogeneration plus gas calcination), so its share of QGP Gladstone-zone deliveries scales near-linearly with output; AIS bauxite arrivals and GPC monthly alumina tonnages anchor the level. Rio has announced a ~40% output cut from October 2026 - detecting the actual ramp-down date and depth is the immediate use case." + required_resolution: 1d + required_accuracy: ±10% + commercial_value: high + strategic_value: high + instrumentation_options: + - { kind: customer_api, notes: "Rio Tinto data-sharing agreement (sole owner-operator)", indicative_cost_band: low } + - { kind: satellite_thermal, notes: "thermal signature of the calciner stacks distinguishes 1-, 2- and 3-calciner operation on each satellite pass", indicative_cost_band: medium } + - { kind: partner_feed, notes: "GPC per-wharf alumina tonnage at daily cadence", indicative_cost_band: medium } + estimated_complexity: medium + priority: P2 + status: open + notes: The announced curtailment makes Yarwun the most likely near-term structural change in Gladstone gas and electricity demand. + + - gap_id: gap:au-nem:orica-yarwun-operating-state + entity_id: chem:au-qld:orica-yarwun + desired_metric: plant.operating.state + current_source: null + best_available_proxy: + source_id: source:aisstream:websocket + metric: vessel.movement.events + derivation: "The site imports ammonia and caustic soda through Fisherman's Landing; ammonia-carrier call cadence (parcels every few weeks) bounds feedstock availability, while step-changes in QGP Gladstone-zone gas deliveries flag trips of the nitric-acid/sodium-cyanide units. Absence of an expected ammonia call implies a turnaround or outage within ~2-4 weeks." + required_resolution: 1d + required_accuracy: state-accurate (running/reduced/down) + commercial_value: medium + strategic_value: high + instrumentation_options: + - { kind: environmental_sensor, notes: "NOx/plume sensing downwind of the three nitric-acid plants gives a direct on/off signature", indicative_cost_band: medium } + - { kind: partner_feed, notes: "Orica supply-chain data share; their AN dispatch schedule is the ground truth", indicative_cost_band: low } + - { kind: camera_ocr, notes: "skyline camera reading steam-plume state of the AN prill towers", indicative_cost_band: low } + estimated_complexity: medium + priority: P3 + status: open + notes: Yarwun produces 500 kt/yr class ammonium nitrate (three nitric-acid plants, two AN plants) plus sodium cyanide; it is the explosives feedstock chokepoint for Bowen Basin mining, so its state propagates into coal-production forecasts. + + - gap_id: gap:au-nem:rg-tanna-shiploader-throughput + entity_id: terminal:au-qld:rg-tanna + desired_metric: bulk.shiploading.tph + current_source: null + best_available_proxy: + source_id: source:aisstream:websocket + metric: vessel.draught.delta + derivation: "AIS draught delta between arrival and departure times tonnes-per-cm immersion (~65-70 t/cm Panamax, ~100-120 t/cm capesize) estimates each parcel to ±5-10%; berth-polygon dwell times give loading intervals and hence average tph per shiploader; monthly reconciliation against GPC cargo statistics removes drift." + required_resolution: 1h + required_accuracy: ±5% + commercial_value: high + strategic_value: high + instrumentation_options: + - { kind: camera_ocr, notes: "camera on the shiploader aprons with OCR of hull names and boom-position detection turns berth state into a live loading signal", indicative_cost_band: medium } + - { kind: partner_feed, notes: "GPC operational feed (terminal is GPC-owned and operated) at daily or shift cadence", indicative_cost_band: low } + - { kind: ais_receiver, notes: "owned AIS receiver at Gladstone removes aggregator gaps and gives <10 s berth-event latency", indicative_cost_band: low } + estimated_complexity: low + priority: P1 + status: open + notes: Gladstone's principal coal terminal - GPC quotes capacity in excess of 60 Mtpa while port-wide coal exports run ~55-70 Mt/yr, of which RG Tanna handles the large majority. Its loading rate is the highest-frequency public-ish observable of the whole Blackwater/Moura coal chain. + + - gap_id: gap:au-nem:wicet-throughput + entity_id: terminal:au-qld:wicet + desired_metric: bulk.throughput.tpd + current_source: null + best_available_proxy: + source_id: source:aisstream:websocket + metric: vessel.draught.delta + derivation: "Single berth and single shiploader make WICET the easiest terminal to observe by AIS - one vessel at a time, parcel size from draught delta, throughput from parcel-per-dwell arithmetic; monthly totals reconciled to GPC cargo statistics." + required_resolution: 1d + required_accuracy: ±5% + commercial_value: medium + strategic_value: medium + instrumentation_options: + - { kind: ais_receiver, notes: "shared with the Gladstone receiver; WICET berth polygon is unambiguous", indicative_cost_band: low } + - { kind: partner_feed, notes: "WICET operational feed via its shipper-owners", indicative_cost_band: medium } + estimated_complexity: low + priority: P2 + status: open + notes: Opened 2015 at 27 Mtpa stage-1 nameplate but debt-burdened and historically well under capacity (~10-15 Mt/yr); utilisation swings are a sensitive read on Bowen Basin export economics at the margin. + + - gap_id: gap:au-nem:barney-point-throughput + entity_id: terminal:au-qld:barney-point + desired_metric: bulk.throughput.tpd + current_source: null + best_available_proxy: + source_id: source:gpc:cargo-statistics + metric: cargo.tonnage.monthly + derivation: "GPC monthly cargo statistics by berth/commodity, ~4-6 weeks in arrears, plus AIS calls at the Barney Point berth for event-level timing." + required_resolution: 1d + required_accuracy: ±10% + commercial_value: low + strategic_value: low + instrumentation_options: + - { kind: ais_receiver, notes: "covered by the same owned Gladstone AIS receiver at no marginal cost", indicative_cost_band: low } + estimated_complexity: low + priority: P4 + status: open + notes: Coal exports ceased in 2016 when GPC consolidated coal at RG Tanna; Barney Point now handles minor dry-bulk trades. Tracked mainly for completeness and as spare-capacity context for the port. + + - gap_id: gap:au-nem:qclng-train-production + entity_id: lng:au-qld:qclng + desired_metric: lng.train.production.tpd + current_source: null + best_available_proxy: + source_id: source:aemo:gas-bulletin-board + metric: pipeline.deliveries.tj + derivation: "Wallumbilla-Gladstone Pipeline daily deliveries to Curtis Island (D+1) divided by ~52-54 TJ per kt of LNG (including ~8-9% liquefaction fuel/shrinkage) gives plant-level output; AIS tanker departures (~65-75 kt LNG per standard 170000 m3 carrier, roughly one every 2-3 days at full rates) confirm. The two 4.25 Mtpa trains share one feed, so per-train attribution is only possible at outage events." + required_resolution: 1d + required_accuracy: ±5% + commercial_value: high + strategic_value: high + instrumentation_options: + - { kind: satellite_thermal, notes: "flare and train heat signatures on each pass separate one-train from two-train operation - the attribution the pipeline proxy cannot do", indicative_cost_band: medium } + - { kind: ais_receiver, notes: "owned Gladstone receiver timestamps loadings at the QCLNG jetty precisely", indicative_cost_band: low } + - { kind: customer_api, notes: "data-sharing agreement with Shell/QGC", indicative_cost_band: high } + estimated_complexity: medium + priority: P2 + status: open + notes: QCLNG - two trains, 8.5 Mtpa nameplate, Shell-operated. Combined Curtis Island feed hit records above 4300 TJ/d in 2024; per-train state is the missing term for east-coast gas balance nowcasts. + + - gap_id: gap:au-nem:aplng-train-production + entity_id: lng:au-qld:aplng + desired_metric: lng.train.production.tpd + current_source: null + best_available_proxy: + source_id: source:aemo:gas-bulletin-board + metric: pipeline.deliveries.tj + derivation: "APLNG Pipeline daily deliveries (D+1) divided by ~52-54 TJ per kt of LNG; AIS tanker cadence at the APLNG jetty confirms. Two 4.5 Mtpa trains (largest of the three projects at 9.0 Mtpa combined) share one feed, so train-level split needs an independent signal." + required_resolution: 1d + required_accuracy: ±5% + commercial_value: high + strategic_value: high + instrumentation_options: + - { kind: satellite_thermal, notes: "train-level thermal/flare discrimination per pass", indicative_cost_band: medium } + - { kind: ais_receiver, notes: "shared Gladstone receiver", indicative_cost_band: low } + - { kind: customer_api, notes: "data-sharing agreement with ConocoPhillips (downstream operator) or Origin (upstream)", indicative_cost_band: high } + estimated_complexity: medium + priority: P2 + status: open + notes: APLNG is the largest domestic-market supplier among the three projects, so its train state moves both LNG exports and the domestic gas balance. + + - gap_id: gap:au-nem:glng-train-production + entity_id: lng:au-qld:glng + desired_metric: lng.train.production.tpd + current_source: null + best_available_proxy: + source_id: source:aemo:gas-bulletin-board + metric: pipeline.deliveries.tj + derivation: "GLNG Gas Transmission Pipeline daily deliveries (D+1) divided by ~52-54 TJ per kt of LNG; AIS tanker cadence confirms. Two trains, 7.8 Mtpa combined nameplate, chronically underfed from equity gas - GLNG's third-party gas purchases make its run-rate the marginal driver of east-coast domestic gas tightness." + required_resolution: 1d + required_accuracy: ±5% + commercial_value: critical + strategic_value: high + instrumentation_options: + - { kind: satellite_thermal, notes: "train-level thermal/flare discrimination; GLNG single-train periods are common and commercially significant", indicative_cost_band: medium } + - { kind: ais_receiver, notes: "shared Gladstone receiver", indicative_cost_band: low } + - { kind: customer_api, notes: "data-sharing agreement with Santos (operator)", indicative_cost_band: high } + estimated_complexity: medium + priority: P1 + status: open + notes: Of the three Curtis Island projects, GLNG's utilisation has the highest information value per tonne because it is the marginal buyer in the domestic gas market. + + - gap_id: gap:au-nem:gladstone-ps-coal-stockpile + entity_id: gen:au-nem:gladstone-ps + desired_metric: fuel.stockpile.tonnes + current_source: null + best_available_proxy: + source_id: source:aemo:nemweb-dispatch-scada + metric: unit.generation.mw + derivation: "Integrate GSTONE1-6 dispatch SCADA at ~0.45-0.55 t coal per MWh (six 280 MW subcritical units, ~4 Mt/yr burn at high utilisation) to get draw-down; rail deliveries are unobserved so the estimate drifts and must be re-anchored by periodic satellite stockpile volumetrics." + required_resolution: 1d + required_accuracy: ±10% + commercial_value: medium + strategic_value: high + instrumentation_options: + - { kind: satellite_optical, notes: "stockpile volumetrics from stereo/pushbroom imagery, weekly tasking, ±10% per measurement - directly re-anchors the burn-integration drift", indicative_cost_band: low } + - { kind: camera_ocr, notes: "camera on the stacker/reclaimer line-of-sight for daily pile-height reads", indicative_cost_band: medium } + - { kind: customer_api, notes: "data share with NRG (operator) or the owner consortium including Rio Tinto (~42%)", indicative_cost_band: medium } + estimated_complexity: medium + priority: P3 + status: open + notes: 1680 MW station whose ~60% contracted supply to Boyne makes its fuel security a direct input to smelter-continuity risk; low-stockpile periods have historically coincided with QLD1 price stress. + + - gap_id: gap:au-nem:gladstone-vessel-queue + entity_id: port:au-qld:gladstone + desired_metric: port.queue.count + current_source: null + best_available_proxy: + source_id: source:aisstream:websocket + metric: vessel.position.count + derivation: "Count vessels inside the designated Gladstone anchorage polygons via aggregated AIS; aggregator coverage of the outer anchorages is patchy (terrestrial receiver range) and position cadence degrades to minutes, so the count is right to ±1-2 vessels most of the time." + required_resolution: 5min + required_accuracy: exact count + commercial_value: high + strategic_value: medium + instrumentation_options: + - { kind: ais_receiver, notes: "one owned AIS receiver on elevated ground at Gladstone covers harbour plus outer anchorages with <10 s latency and closes the gap outright; also serves every other Gladstone AIS-proxied gap", indicative_cost_band: low } + estimated_complexity: low + priority: P2 + status: proxied + notes: Queue depth leads terminal throughput by days and prices demurrage risk across coal, LNG and alumina trades; this is the single highest-leverage owned-sensor deployment in the cluster. diff --git a/intelligence/data/registries/gaps/au-nem-other.yaml b/intelligence/data/registries/gaps/au-nem-other.yaml new file mode 100644 index 0000000..5c7e783 --- /dev/null +++ b/intelligence/data/registries/gaps/au-nem-other.yaml @@ -0,0 +1,350 @@ +# Observability-gap registry — NEM outside the Gladstone cluster (Bowen Basin, Hunter, Sydney, statewide QLD/NSW). See ../../schemas/observability-gap.schema.json +records: + - gap_id: gap:au-nem:blackwater-mine-production + entity_id: mine:au-qld:blackwater + desired_metric: coal.production.tpd + current_source: null + best_available_proxy: + source_id: source:gpc:cargo-statistics + metric: cargo.tonnage.monthly + derivation: "Monthly GPC coal tonnages by terminal, allocated to Blackwater by known offtake/berth patterns and reconciled against Whitehaven quarterly production reports and the GEM tracker's 16 Mtpa capacity figure; AIS parcel-level data at RG Tanna/WICET sharpens timing to ~weekly. No public feed attributes cargoes to mines, so allocation error dominates." + required_resolution: 1d + required_accuracy: ±10% + commercial_value: high + strategic_value: high + instrumentation_options: + - { kind: partner_feed, notes: "Aurizon Network railings-by-load-point feed is the single dataset that closes every Bowen Basin mine gap at once", indicative_cost_band: medium } + - { kind: gnss, notes: "GNSS trackers on contracted rakes serving the mine's balloon loop count loaded services directly", indicative_cost_band: medium } + - { kind: camera_ocr, notes: "camera at the train load-out reading rake IDs and counting wagons", indicative_cost_band: low } + - { kind: customer_api, notes: "data-sharing agreement with Whitehaven (owner since April 2024 BHP/BMA divestment)", indicative_cost_band: low } + estimated_complexity: medium + priority: P2 + status: open + notes: One of the southern hemisphere's largest open-cut coal mines (~12-14 Mt/yr recent, 16 Mtpa capacity, ~80 km strike); anchor tonnage of the Blackwater rail system and RG Tanna. + + - gap_id: gap:au-nem:goonyella-riverside-production + entity_id: mine:au-qld:goonyella-riverside + desired_metric: coal.production.tpd + current_source: null + best_available_proxy: + source_id: source:aisstream:websocket + metric: vessel.draught.delta + derivation: "Exports via Hay Point/DBCT (outside GPC statistics); AIS draught-delta parcels at those terminals allocated to BMA berths, calibrated against BHP quarterly reports and the GEM tracker (~15-16 Mtpa met coal). Mine-level attribution is the weak link - Goonyella-system tonnage is observable, this mine's share is modelled." + required_resolution: 1d + required_accuracy: ±15% + commercial_value: high + strategic_value: high + instrumentation_options: + - { kind: partner_feed, notes: "Aurizon railings-by-load-point feed (same agreement as Blackwater)", indicative_cost_band: medium } + - { kind: gnss, notes: "trackers on rakes serving the Goonyella/Riverside load-outs", indicative_cost_band: medium } + - { kind: customer_api, notes: "data-sharing agreement with BMA (BHP Mitsubishi Alliance)", indicative_cost_band: low } + estimated_complexity: medium + priority: P2 + status: open + notes: Flagship BMA metallurgical mine on the Goonyella system; with Peak Downs it sets the tempo of the world's largest seaborne met-coal artery. + + - gap_id: gap:au-nem:peak-downs-production + entity_id: mine:au-qld:peak-downs + desired_metric: coal.production.tpd + current_source: null + best_available_proxy: + source_id: source:gem:global-coal-mine-tracker + metric: mine.capacity.mtpa + derivation: "Annual GEM capacity (~11-12 Mtpa hard coking) scaled by BHP quarterly actuals; intra-quarter state is inferred from the same Goonyella-system AIS/rail signals as Goonyella Riverside. Effectively quarterly observability today." + required_resolution: 1d + required_accuracy: ±15% + commercial_value: medium + strategic_value: high + instrumentation_options: + - { kind: partner_feed, notes: "Aurizon railings-by-load-point feed", indicative_cost_band: medium } + - { kind: gnss, notes: "rake trackers via haulage partner", indicative_cost_band: medium } + - { kind: customer_api, notes: "BMA data-sharing agreement", indicative_cost_band: low } + estimated_complexity: medium + priority: P3 + status: open + notes: Premium hard-coking producer; same instrumentation pattern as Goonyella Riverside, one contract covers both. + + - gap_id: gap:au-nem:curragh-production + entity_id: mine:au-qld:curragh + desired_metric: coal.production.tpd + current_source: null + best_available_proxy: + source_id: source:aemo:nemweb-dispatch-scada + metric: unit.generation.mw + derivation: "Curragh (Coronado, ~11-13 Mt/yr ROM) has a dual signature - export met coal via the Blackwater system to RG Tanna (GPC monthly + AIS parcels) and a long-term domestic thermal contract to Stanwell Power Station, whose STAN-1..4 dispatch SCADA converts at ~0.4-0.45 t/MWh into observed domestic offtake. Combining the two bounds total production to ~±10% monthly." + required_resolution: 1d + required_accuracy: ±10% + commercial_value: medium + strategic_value: high + instrumentation_options: + - { kind: partner_feed, notes: "Aurizon railings-by-load-point feed", indicative_cost_band: medium } + - { kind: camera_ocr, notes: "load-out camera at the Curragh balloon loop", indicative_cost_band: low } + - { kind: customer_api, notes: "Coronado data-sharing agreement", indicative_cost_band: low } + estimated_complexity: medium + priority: P2 + status: open + notes: The Stanwell supply contract couples this mine directly into QLD1 generation adequacy, making its state relevant to both coal-market and power-market products. + + - gap_id: gap:au-nem:callide-mine-production + entity_id: mine:au-qld:callide + desired_metric: coal.production.tpd + current_source: null + best_available_proxy: + source_id: source:aemo:nemweb-dispatch-scada + metric: unit.generation.mw + derivation: "Mine-mouth supplier to Callide B/C power stations plus exports via the Moura system; Callide unit dispatch SCADA at ~0.5 t/MWh gives the domestic draw in near-real-time, GPC monthly tonnage gives the export residual. Batchfire capacity ~10 Mtpa ROM." + required_resolution: 1d + required_accuracy: ±10% + commercial_value: medium + strategic_value: medium + instrumentation_options: + - { kind: partner_feed, notes: "Aurizon Moura-system railings feed", indicative_cost_band: medium } + - { kind: customer_api, notes: "Batchfire data-sharing agreement", indicative_cost_band: low } + estimated_complexity: low + priority: P3 + status: open + notes: The mine-mouth coupling means Callide C outage/return schedules (post-2021 explosion rebuild) dominate this mine's demand profile; power-side SCADA already reveals most of its state. + + - gap_id: gap:au-nem:blackwater-system-consists + entity_id: rail:au-qld:blackwater-system + desired_metric: rail.consist.positions + current_source: null + best_available_proxy: + source_id: source:aisstream:websocket + metric: vessel.loading.events + derivation: "No public feed carries Aurizon freight positions (QLD GTFS-Realtime covers passenger services only). Terminal-side shiploading at RG Tanna/WICET back-casts delivered railings through a ~2-5 day stockyard buffer; ~15-20 loaded electric services/day at ~8-10 kt payload on the electrified 25 kV system when running at its ~50-60 Mt/yr rate." + required_resolution: 5min + required_accuracy: ±1 consist + commercial_value: high + strategic_value: high + instrumentation_options: + - { kind: partner_feed, notes: "Aurizon Network operational feed - the definitive closure for all three system gaps", indicative_cost_band: high } + - { kind: gnss, notes: "GNSS units on rakes via a haulage or maintenance partner", indicative_cost_band: medium } + - { kind: camera_ocr, notes: "trackside cameras at Callemondah yard throat and the Rocklands junction counting and identifying consists", indicative_cost_band: medium } + - { kind: condition_sensor, notes: "trackside vibration/axle-counter nodes at 2-3 chokepoints give direction and consist length without imagery", indicative_cost_band: low } + estimated_complexity: high + priority: P2 + status: open + notes: The Gladstone coal chain's velocity term; a derailment or flood closure here shows up in port data only days later, so direct consist observability buys the lead time. + + - gap_id: gap:au-nem:moura-system-consists + entity_id: rail:au-qld:moura-system + desired_metric: rail.consist.positions + current_source: null + best_available_proxy: + source_id: source:gpc:cargo-statistics + metric: cargo.tonnage.monthly + derivation: "Diesel-hauled system, ~10-13 Mt/yr from Moura/Dawson and Baralaba to Gladstone; monthly GPC tonnage minus Blackwater-system attribution leaves the Moura share, a few loaded services/day." + required_resolution: 15min + required_accuracy: ±1 consist + commercial_value: low + strategic_value: medium + instrumentation_options: + - { kind: partner_feed, notes: "covered by the same Aurizon agreement", indicative_cost_band: high } + - { kind: condition_sensor, notes: "single trackside node near Callemondah suffices for a low-traffic system", indicative_cost_band: low } + estimated_complexity: medium + priority: P3 + status: open + notes: Smallest of the three systems; monitored mainly because Orica AN and Callide export tonnage ride on it. + + - gap_id: gap:au-nem:goonyella-system-consists + entity_id: rail:au-qld:goonyella-system + desired_metric: rail.consist.positions + current_source: null + best_available_proxy: + source_id: source:aisstream:websocket + metric: vessel.loading.events + derivation: "Hay Point and DBCT shiploading cadence back-casts system railings through the terminal stockyards; the electrified ~924 km Goonyella system serves ~30 mines and moves ~100-120 Mt/yr - the largest export met-coal artery in the world - at ~25-35 loaded services/day." + required_resolution: 5min + required_accuracy: ±1 consist + commercial_value: high + strategic_value: high + instrumentation_options: + - { kind: partner_feed, notes: "Aurizon Network operational feed", indicative_cost_band: high } + - { kind: gnss, notes: "rake trackers via haulage partner", indicative_cost_band: medium } + - { kind: camera_ocr, notes: "trackside cameras at the Coppabella junction and Jilalan yard", indicative_cost_band: medium } + estimated_complexity: high + priority: P2 + status: open + notes: Weather (cyclone/flood) interruptions on this corridor move global met-coal prices; consist-level observability converts BoM rainfall into tonnage-at-risk within hours instead of weeks. + + - gap_id: gap:au-nem:tomago-live-load + entity_id: smelter:au-nsw:tomago + desired_metric: power.load.mw + current_source: null + best_available_proxy: + source_id: source:aemo:nemweb-dispatch-scada + metric: grid.demand.mw + derivation: "NSW1 regional demand net of scheduled generation; Tomago draws a near-constant ~950 MW (~10-12% of NSW consumption, ~8300 GWh/yr) and its potline interruptions during price spikes (up to ~75 min per potline) appear as sharp ~300 MW residual-demand steps that identify it uniquely." + required_resolution: 1min + required_accuracy: ±1% + commercial_value: high + strategic_value: critical + instrumentation_options: + - { kind: substation_gateway, notes: "Tomago is fed by dedicated 330 kV circuits from the Tomago substation; a gateway there gives exact potline load", indicative_cost_band: high } + - { kind: revenue_meter, notes: "retailer/TNSP revenue-meter integration via customer API", indicative_cost_band: medium } + - { kind: customer_api, notes: "direct data-sharing agreement with Tomago Aluminium (Rio Tinto-majority JV)", indicative_cost_band: low } + estimated_complexity: medium + priority: P1 + status: open + notes: Australia's largest single electricity consumer (590 kt/yr aluminium); the NSW1 analogue of Boyne, and the demand-response asset that most often sets NSW scarcity outcomes. + + - gap_id: gap:au-nem:port-kembla-operating-state + entity_id: steel:au-nsw:port-kembla + desired_metric: plant.operating.state + current_source: null + best_available_proxy: + source_id: source:aisstream:websocket + metric: vessel.movement.events + derivation: "Raw-material vessel cadence at the Port Kembla ore berths - ~3.1 Mt/yr crude steel needs ~4.5 Mt/yr of imported iron ore and fluxes, roughly one Panamax-class arrival per 5-7 days - plus outbound slab/coil sailings; a widening arrival gap implies reduced blast-furnace burden within ~2 weeks." + required_resolution: 1d + required_accuracy: state-accurate (running/reduced/down) + commercial_value: medium + strategic_value: high + instrumentation_options: + - { kind: satellite_thermal, notes: "blast-furnace and coke-oven thermal signatures give a direct daily on/off/banked read - the canonical steel-plant instrument", indicative_cost_band: low } + - { kind: partner_feed, notes: "BlueScope data-sharing agreement", indicative_cost_band: medium } + - { kind: environmental_sensor, notes: "downwind particulate/SO2 sensing correlates with sinter and BF operation", indicative_cost_band: medium } + estimated_complexity: low + priority: P3 + status: open + notes: Single-blast-furnace configuration (No.6 relined for ~$1.15b, replacing No.5 by mid-2026) means plant state is near-binary and any outage is nationally significant - Port Kembla is one of only two primary steelmaking sites in Australia. + + - gap_id: gap:au-nem:newcastle-coal-chain-throughput + entity_id: port:au-nsw:newcastle + desired_metric: bulk.throughput.tpd + current_source: null + best_available_proxy: + source_id: source:aisstream:websocket + metric: vessel.draught.delta + derivation: "Departure draught deltas per vessel across the three terminals (PWCS Kooragang + Carrington 145 Mtpa, NCIG 79 Mtpa; combined ~224 Mtpa against ~135-150 Mt/yr actual exports) plus the famous Newcastle anchorage queue count give daily throughput to ~±5%; the AIS picture at Newcastle is unusually complete because all berths and anchorages sit inside dense terrestrial coverage." + required_resolution: 1d + required_accuracy: ±5% + commercial_value: high + strategic_value: high + instrumentation_options: + - { kind: ais_receiver, notes: "owned receiver at Newcastle removes aggregator dependency and gives sub-minute berth events", indicative_cost_band: low } + - { kind: partner_feed, notes: "PWCS/NCIG or HVCCC daily operational reports under agreement", indicative_cost_band: medium } + estimated_complexity: low + priority: P2 + status: proxied + notes: The world's largest coal-export port; the AIS proxy is good enough that this gap is accepted as proxied, with owned instrumentation only hardening latency and provenance. + + - gap_id: gap:au-nem:hvo-production + entity_id: mine:au-nsw:hunter-valley-operations + desired_metric: coal.production.tpd + current_source: null + best_available_proxy: + source_id: source:gem:global-coal-mine-tracker + metric: mine.capacity.mtpa + derivation: "GEM annual capacity (~13 Mtpa-class saleable thermal, Yancoal 51/Glencore 49 JV) scaled by owners' quarterly reports; intra-quarter state is inferred from Newcastle chain throughput, but ~35 mines share that signal so mine-level attribution is weak." + required_resolution: 1d + required_accuracy: ±15% + commercial_value: medium + strategic_value: medium + instrumentation_options: + - { kind: partner_feed, notes: "ARTC Hunter Valley network or HVCCC railings-by-load-point feed", indicative_cost_band: medium } + - { kind: gnss, notes: "rake trackers via haulage partner (Pacific National/Aurizon)", indicative_cost_band: medium } + - { kind: customer_api, notes: "Yancoal/Glencore data-sharing agreement", indicative_cost_band: low } + estimated_complexity: medium + priority: P3 + status: open + notes: Representative Hunter mine record - the instrumentation pattern (rail partner feed) generalises to the whole Hunter Valley coal chain exactly as the Aurizon pattern does for the Bowen Basin. + + - gap_id: gap:au-nem:sydney-dc-cluster-load + entity_id: dc:au-nsw:sydney-metro-cluster + desired_metric: power.load.mw + current_source: null + best_available_proxy: null + required_resolution: 5min + required_accuracy: ±5% + commercial_value: high + strategic_value: high + instrumentation_options: + - { kind: customer_api, notes: "data-sharing agreements with operators (NextDC, AirTrunk, CDC, Equinix and others) - even 2-3 anchor tenants would calibrate a cluster model", indicative_cost_band: medium } + - { kind: partner_feed, notes: "DNSP (Ausgrid/Endeavour) zone-substation feed under a data-services agreement; the public versions are annual and 12-18 months in arrears", indicative_cost_band: high } + - { kind: power_quality_meter, notes: "owned meters at a sample of campus connection points under host agreements", indicative_cost_band: high } + estimated_complexity: high + priority: P2 + status: open + notes: A pure gap - no public source resolves data-centre load at any cadence. Sydney is Australia's largest DC market (~770 MW built 2025, >1 GW during 2026, ~1.4 GW by 2029) and the fastest-growing structural component of NSW1 demand; today it is invisible inside regional operational demand. + + - gap_id: gap:au-nem:wivenhoe-storage-state + entity_id: gen:au-nem:wivenhoe + desired_metric: storage.energy.mwh + current_source: null + best_available_proxy: + source_id: source:aemo:nemweb-dispatch-scada + metric: unit.generation.mw + derivation: "Integrate Wivenhoe generation (2x285 MW units) minus pumping loads (both visible as scheduled DUIDs in dispatch SCADA) at ~70-75% round-trip efficiency to track Splityard Creek state-of-charge (~5 GWh-class usable, ~8-9 h at full output); re-anchor whenever sustained max-generation runs imply a full or empty reservoir. Drift ~5-10% per cycle." + required_resolution: 1h + required_accuracy: ±5% + commercial_value: medium + strategic_value: high + instrumentation_options: + - { kind: satellite_optical, notes: "shoreline waterline extraction on Splityard Creek per pass gives absolute level to ~0.5 m", indicative_cost_band: low } + - { kind: customer_api, notes: "CS Energy or Seqwater data share for reservoir level", indicative_cost_band: low } + - { kind: camera_ocr, notes: "fixed camera reading the reservoir staff gauge", indicative_cost_band: low } + estimated_complexity: low + priority: P3 + status: proxied + notes: Queensland's only operating pumped storage until Borumba; its state-of-charge is the QLD1 evening-peak insurance variable, and the pump/generate pattern is a live signal of expected price spreads. + + - gap_id: gap:au-nem:qld-rooftop-pv-output + entity_id: region:au-nem:qld1 + desired_metric: solar.rooftop.mw + current_source: source:openelectricity:api + best_available_proxy: + source_id: source:openelectricity:api + metric: solar.rooftop.estimate.mw + derivation: "AEMO's half-hourly distributed-PV estimate (satellite-irradiance plus sample-meter methods, surfaced through the OpenElectricity API) for QLD1's >5 GW of installed rooftop capacity; it is a model, not a measurement - errors of ±5-10% during cloud-transient periods are exactly when the number matters for minimum-demand operations." + required_resolution: 5min + required_accuracy: ±2% + commercial_value: medium + strategic_value: high + instrumentation_options: + - { kind: iot_gateway, notes: "inverter-fleet telemetry via partnership with a retailer or monitoring platform - a ~1000-site stratified sample measures the fleet to ~1%", indicative_cost_band: medium } + - { kind: weather_station, notes: "owned irradiance stations at 5-6 QLD population centres tighten the satellite model's cloud-edge error", indicative_cost_band: low } + estimated_complexity: medium + priority: P3 + status: proxied + notes: Behind-the-meter PV now sets QLD1 minimum operational demand records; the gap is accuracy and cadence, not existence, hence proxied status with an owned-sample upgrade path. + + - gap_id: gap:au-nem:nem-frequency-4s + entity_id: grid:au-nem + desired_metric: grid.frequency.hz + current_source: null + best_available_proxy: + source_id: source:aemo:nemweb-dispatch-scada + metric: dispatch.interval.mw + derivation: "Nothing public is live - dispatch SCADA is 5-min MW with no frequency, and AEMO's 4-second FCAS/causer-pays files are published in periodic batches weeks in arrears; the proxy is therefore retrospective frequency-performance analysis only." + required_resolution: 4s + required_accuracy: ±10 mHz + commercial_value: medium + strategic_value: medium + instrumentation_options: + - { kind: power_quality_meter, notes: "frequency is uniform across a synchronous grid, so one owned PQ/PMU-class meter on any NEM-mainland socket closes the gap completely; a second unit in TAS covers the other synchronous island - the cheapest full gap closure in this registry", indicative_cost_band: low } + estimated_complexity: low + priority: P3 + status: open + notes: Live frequency unlocks FCAS-event detection, contingency (unit-trip) inference within seconds, and independent verification of AEMO's delayed data - high analytic leverage for trivial capex. + + - gap_id: gap:au-nem:qld1-industrial-load-mix + entity_id: region:au-nem:qld1 + desired_metric: demand.industrial.mw + current_source: null + best_available_proxy: + source_id: source:aemo:nemweb-dispatchis + metric: region.demand.mw + derivation: "Decompose QLD1 operational demand (5-min) into a temperature-driven mass-market component (regressed on BoM observations), the AEMO rooftop-PV estimate, and a residual ~3.5-4 GW industrial baseload band (Boyne, QAL, Yarwun, rail traction, mines); step-changes in the residual attribute to specific facilities via the facility-level gap records above." + required_resolution: 30min + required_accuracy: ±5% + commercial_value: high + strategic_value: high + instrumentation_options: + - { kind: substation_gateway, notes: "gateways at the 3-4 dedicated industrial substations (Boyne, Gladstone South, Nebo) would convert the residual model into measurement", indicative_cost_band: high } + - { kind: customer_api, notes: "each facility-level customer agreement independently hardens one term of the decomposition", indicative_cost_band: medium } + estimated_complexity: medium + priority: P2 + status: proxied + notes: The aggregate anchor for all QLD facility-level records - every individual gap closed tightens this decomposition, and the decomposition cross-checks every individual proxy. diff --git a/intelligence/data/registries/gaps/au-wem-nwis.yaml b/intelligence/data/registries/gaps/au-wem-nwis.yaml new file mode 100644 index 0000000..7366bf7 --- /dev/null +++ b/intelligence/data/registries/gaps/au-wem-nwis.yaml @@ -0,0 +1,80 @@ +# Observability-gap registry — WEM (SWIS) and NWIS, Western Australia. See ../../schemas/observability-gap.schema.json +records: + - gap_id: gap:au-wem:facility-scada-latency + entity_id: grid:au-wem + desired_metric: power.generation.mw + current_source: source:aemo:wem-data-portal + best_available_proxy: + source_id: source:aemo:wem-data-portal + metric: facility.scada.mw + derivation: "The WEM data portal does publish per-facility SCADA for the SWIS (5-minute dispatch intervals since the new WEM commenced October 2023), but as batch files with hours-to-a-day publication lag and no streaming interface - the metric exists, the latency does not meet live-intelligence requirements." + required_resolution: 5min + required_accuracy: ±1% + commercial_value: medium + strategic_value: medium + instrumentation_options: + - { kind: power_quality_meter, notes: "one owned PQ meter anywhere on the SWIS gives real-time frequency and event detection (unit trips, load shedding) during the portal's publication lag", indicative_cost_band: low } + - { kind: partner_feed, notes: "market-participant data share for real-time dispatch targets ahead of publication", indicative_cost_band: medium } + estimated_complexity: low + priority: P3 + status: proxied + notes: The SWIS (~4 GW-class peak) is well covered on a delay; this is a latency gap, not an existence gap, so it is accepted as proxied with a cheap event-detection overlay. + + - gap_id: gap:au-nwis:system-visibility + entity_id: grid:au-nwis + desired_metric: power.demand.mw + current_source: null + best_available_proxy: null + required_resolution: 5min + required_accuracy: ±2% + commercial_value: medium + strategic_value: high + instrumentation_options: + - { kind: power_quality_meter, notes: "an owned PQ meter on the network at Karratha or Port Hedland gives real-time frequency, voltage events and system-stress signatures for the entire islanded grid - the only unilateral instrument available", indicative_cost_band: low } + - { kind: partner_feed, notes: "data agreement with Horizon Power or Pilbara ISOCo (independent system operator since 2022)", indicative_cost_band: medium } + - { kind: customer_api, notes: "data share with a connected miner (Rio Tinto, Alinta-supplied BHP loads) covering their offtake", indicative_cost_band: medium } + estimated_complexity: medium + priority: P2 + status: open + notes: A pure gap - the NWIS publishes NO real-time operational data at all - no dispatch feed, no demand feed, no prices; Pilbara ISOCo releases only planning documents and a reduced-order model. The interconnected Horizon Power/Alinta/Rio Tinto networks meet roughly a fifth of Pilbara electricity needs (hundreds-of-MW scale), with the rest behind private islanded systems; it is the least observable strategically significant grid in Australia. + + - gap_id: gap:au-nwis:mining-load + entity_id: grid:au-nwis + desired_metric: demand.industrial.mw + current_source: null + best_available_proxy: + source_id: source:aemo:gas-bulletin-board + metric: gas.consumption.tj + derivation: "AEMO's WA gas bulletin board publishes large-user and power-station gas consumption in the Pilbara (D+ days); gas burn at the NWIS-connected stations converted at ~7-9 GJ/MWh open-cycle and ~7 GJ/MWh CCGT approximates generation and hence mining load - an estimate that degrades as Pilbara solar/battery build-out (e.g. miners' renewable hubs) displaces gas." + required_resolution: 1h + required_accuracy: ±10% + commercial_value: medium + strategic_value: high + instrumentation_options: + - { kind: partner_feed, notes: "data agreements with the Pilbara miners (BHP, Rio Tinto, Fortescue) or their retailer Alinta for port/rail/plant loads", indicative_cost_band: medium } + - { kind: customer_api, notes: "single-miner pilot (port operations load is the most contractable slice)", indicative_cost_band: low } + - { kind: iot_gateway, notes: "owned gateways at partner substations if a miner hosts them", indicative_cost_band: high } + estimated_complexity: high + priority: P3 + status: open + notes: Iron-ore port and rail electrical load is a direct physical correlate of export run-rate at Port Hedland and Dampier; observing it would give a power-side confirmation channel for AIS-derived iron-ore flow estimates. + + - gap_id: gap:au-wem:industrial-load-mix + entity_id: grid:au-wem + desired_metric: demand.industrial.mw + current_source: null + best_available_proxy: + source_id: source:aemo:wem-data-portal + metric: operational.demand.mw + derivation: "SWIS operational demand decomposed against a temperature model and published facility SCADA for industrial cogeneration exports (Alcoa Pinjarra/Wagerup, Worsley) leaves an industrial band; large events are legible - e.g. the BHP Kwinana nickel refinery suspension (December 2024) - but the split of Kwinana-strip loads is modelled, not measured." + required_resolution: 30min + required_accuracy: ±10% + commercial_value: medium + strategic_value: medium + instrumentation_options: + - { kind: customer_api, notes: "data-sharing agreements with the 3-4 dominant industrial consumers (Alcoa, Worsley/South32, Kwinana strip operators)", indicative_cost_band: medium } + - { kind: substation_gateway, notes: "gateway at the Kwinana terminal substation", indicative_cost_band: high } + estimated_complexity: medium + priority: P4 + status: open + notes: Lower priority than the QLD1 equivalent because SWIS industrial load is smaller and partially self-supplied by behind-the-fence cogeneration; tracked for WA alumina-chain intelligence. From 30d7fbb9bbd36d3813934e65cbda24f12540ebac Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 01:52:41 +0000 Subject: [PATCH 11/16] =?UTF-8?q?feat(intelligence):=20NEM=20structural=20?= =?UTF-8?q?seed=20=E2=80=94=20Gladstone=20cluster=20world=20model=20(phase?= =?UTF-8?q?s=207-13)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 121 entities and 206 relationships: NEM regions, 25 QLD/NSW stations with real DUIDs, six interconnectors, the full Gladstone chain (GPS → Boyne, QAL/ Yarwun alumina, RG Tanna/WICET/Barney Point, Curtis Island LNG trains, Bowen Basin mines and the three Aurizon coal systems), ownership current to 2026 (Sev.en at Callide C, Whitehaven Blackwater JV, APA Basslink, Eraring extension to 2029), commodity flows to country-level destinations, and three clearly-flagged synthetic exemplar vessels. Inferred commercial links use MAY_SUPPLY with confidence < 1; no private contracts fabricated. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014yVGqBtvM1zNAd9tvUVNzm --- .../entities/au-nem/commodities-markets.yaml | 174 +++++ .../data/entities/au-nem/companies.yaml | 468 ++++++++++++ .../data/entities/au-nem/gas-lng.yaml | 150 ++++ .../entities/au-nem/gladstone-cluster.yaml | 155 ++++ .../data/entities/au-nem/interconnectors.yaml | 142 ++++ .../data/entities/au-nem/mines-basins.yaml | 219 ++++++ .../data/entities/au-nem/nsw-generators.yaml | 171 +++++ .../entities/au-nem/ports-terminals-rail.yaml | 214 ++++++ .../data/entities/au-nem/qld-generators.yaml | 443 +++++++++++ .../data/entities/au-nem/regions.yaml | 111 +++ .../data/entities/au-nem/vessels.yaml | 75 ++ .../relationships/au-nem/fuel-supply.yaml | 330 ++++++++ .../data/relationships/au-nem/generation.yaml | 289 +++++++ .../relationships/au-nem/gladstone-chain.yaml | 436 +++++++++++ .../relationships/au-nem/grid-structure.yaml | 272 +++++++ .../data/relationships/au-nem/logistics.yaml | 267 +++++++ .../data/relationships/au-nem/ownership.yaml | 723 ++++++++++++++++++ 17 files changed, 4639 insertions(+) create mode 100644 intelligence/data/entities/au-nem/commodities-markets.yaml create mode 100644 intelligence/data/entities/au-nem/companies.yaml create mode 100644 intelligence/data/entities/au-nem/gas-lng.yaml create mode 100644 intelligence/data/entities/au-nem/gladstone-cluster.yaml create mode 100644 intelligence/data/entities/au-nem/interconnectors.yaml create mode 100644 intelligence/data/entities/au-nem/mines-basins.yaml create mode 100644 intelligence/data/entities/au-nem/nsw-generators.yaml create mode 100644 intelligence/data/entities/au-nem/ports-terminals-rail.yaml create mode 100644 intelligence/data/entities/au-nem/qld-generators.yaml create mode 100644 intelligence/data/entities/au-nem/regions.yaml create mode 100644 intelligence/data/entities/au-nem/vessels.yaml create mode 100644 intelligence/data/relationships/au-nem/fuel-supply.yaml create mode 100644 intelligence/data/relationships/au-nem/generation.yaml create mode 100644 intelligence/data/relationships/au-nem/gladstone-chain.yaml create mode 100644 intelligence/data/relationships/au-nem/grid-structure.yaml create mode 100644 intelligence/data/relationships/au-nem/logistics.yaml create mode 100644 intelligence/data/relationships/au-nem/ownership.yaml diff --git a/intelligence/data/entities/au-nem/commodities-markets.yaml b/intelligence/data/entities/au-nem/commodities-markets.yaml new file mode 100644 index 0000000..eea45fe --- /dev/null +++ b/intelligence/data/entities/au-nem/commodities-markets.yaml @@ -0,0 +1,174 @@ +# Structural seed — commodities, export destinations and demand markets. +# See ../../../schemas/entity.schema.json +records: + - entity_id: commodity:coal-thermal + entity_type: Commodity + name: Thermal Coal + aliases: [steam coal] + data_class: structural + observability: UNCLASSIFIED + geometry: null + country: null + grid_id: null + properties: { benchmark: Newcastle 6000 kcal (GCNewc), unit: tonne } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Power-station coal; Australian benchmark is the Newcastle 6,000 kcal/kg spot assessment. + + - entity_id: commodity:coal-metallurgical + entity_type: Commodity + name: Metallurgical Coal + aliases: [coking coal, met coal] + data_class: structural + observability: UNCLASSIFIED + geometry: null + country: null + grid_id: null + properties: { benchmark: Premium Low-Vol HCC FOB Australia, unit: tonne } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Steelmaking coal (hard coking, semi-soft, PCI); Bowen Basin is the world's dominant seaborne source. + + - entity_id: commodity:lng + entity_type: Commodity + name: Liquefied Natural Gas + aliases: [LNG] + data_class: structural + observability: UNCLASSIFIED + geometry: null + country: null + grid_id: null + properties: { benchmark: JKM / oil-linked term contracts, unit: tonne } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Curtis Island exports are mostly oil-linked term volumes with spot cargoes at the margin. + + - entity_id: commodity:alumina + entity_type: Commodity + name: Alumina + aliases: [aluminium oxide] + data_class: structural + observability: UNCLASSIFIED + geometry: null + country: null + grid_id: null + properties: { benchmark: Platts Alumina Index FOB Australia, unit: tonne } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Intermediate refined from bauxite; ~2 t alumina per tonne of aluminium. + + - entity_id: commodity:aluminium + entity_type: Commodity + name: Aluminium + aliases: [primary aluminium] + data_class: structural + observability: UNCLASSIFIED + geometry: null + country: null + grid_id: null + properties: { benchmark: LME aluminium, unit: tonne } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Electricity-intensive primary metal (~14 MWh/t); Boyne Island and Tomago are the NEM's two smelters. + + - entity_id: commodity:bauxite + entity_type: Commodity + name: Bauxite + aliases: [] + data_class: structural + observability: UNCLASSIFIED + geometry: null + country: null + grid_id: null + properties: { unit: tonne } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Aluminium ore; Rio Tinto ships Weipa (Cape York) bauxite to the Gladstone refineries. + + - entity_id: dest:jp + entity_type: Destination + name: Japan + aliases: [] + data_class: structural + observability: UNCLASSIFIED + geometry: { type: Point, coordinates: [138.25, 36.2] } + country: jp + grid_id: null + properties: { role: export destination } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Largest long-term buyer of Australian metallurgical coal and LNG; major aluminium importer. + + - entity_id: dest:kr + entity_type: Destination + name: South Korea + aliases: [Republic of Korea] + data_class: structural + observability: UNCLASSIFIED + geometry: { type: Point, coordinates: [127.8, 36.5] } + country: kr + grid_id: null + properties: { role: export destination } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Major buyer of Australian coal, LNG (KOGAS is a GLNG partner) and aluminium. + + - entity_id: dest:cn + entity_type: Destination + name: China + aliases: [People's Republic of China] + data_class: structural + observability: UNCLASSIFIED + geometry: { type: Point, coordinates: [104.2, 35.9] } + country: cn + grid_id: null + properties: { role: export destination } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Dominant LNG destination for QCLNG/APLNG term volumes (CNOOC, Sinopec); coal flows resumed after the 2020-23 informal ban. + + - entity_id: dest:tw + entity_type: Destination + name: Taiwan + aliases: [] + data_class: structural + observability: UNCLASSIFIED + geometry: { type: Point, coordinates: [121.0, 23.7] } + country: tw + grid_id: null + properties: { role: export destination } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Steady buyer of Australian thermal and metallurgical coal. + + - entity_id: dest:in + entity_type: Destination + name: India + aliases: [] + data_class: structural + observability: UNCLASSIFIED + geometry: { type: Point, coordinates: [78.96, 20.59] } + country: in + grid_id: null + properties: { role: export destination } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Largest growth buyer of Australian metallurgical coal for its expanding steel sector. + + - entity_id: market:jp:steel + entity_type: Market + name: Japanese Steel Industry + aliases: [Japan steel market] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [139.7, 35.7] } + country: jp + grid_id: null + properties: + demand_note: ~85 Mtpa crude steel (Nippon Steel, JFE, Kobe); blast-furnace route drives coking coal imports + external_ids: {} + sources: [source:wikidata:sparql] + notes: >- + Demand-side market entity for the metallurgical coal chain. Nippon Steel and JFE's 2025 equity entry + into the Blackwater mine makes the structural linkage explicit. Monthly output visible via public + steel statistics. diff --git a/intelligence/data/entities/au-nem/companies.yaml b/intelligence/data/entities/au-nem/companies.yaml new file mode 100644 index 0000000..7f96ff6 --- /dev/null +++ b/intelligence/data/entities/au-nem/companies.yaml @@ -0,0 +1,468 @@ +# Structural seed — companies and operators referenced by ownership/operation relationships. +# See ../../../schemas/entity.schema.json. Companies are modelled as their Australian operating presence; +# multinational parents noted in notes. Geometry null (non-spatial). +records: + - entity_id: company:au:rio-tinto + entity_type: Company + name: Rio Tinto + aliases: [Rio Tinto Group] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: mining and metals, listed: true, ticker: RIO } + external_ids: { wikidata_qid: Q206423 } + sources: [source:wikidata:sparql] + notes: Dual-listed UK/Australian miner. Anchor of the Gladstone aluminium chain - Boyne Smelters ~73.5%, QAL 80%, Yarwun 100%, Gladstone PS 42.125%, Tomago ~51.55%, plus Weipa bauxite. + + - entity_id: company:au:nrg-gladstone + entity_type: Company + name: NRG Gladstone Operating Services + aliases: [NRG Gladstone] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: power generation operations } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Australian subsidiary of NRG Energy Inc (US); operator of Gladstone Power Station for the JV. NRG Energy also holds a 37.5% JV interest, represented on this entity. + + - entity_id: company:au:cs-energy + entity_type: Company + name: CS Energy + aliases: [] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: power generation, ownership: Queensland Government } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Queensland government-owned generator - Callide B, 50% of Callide C (operator), Kogan Creek station and mine; also market intermediary for Gladstone PS output. + + - entity_id: company:au:stanwell-corp + entity_type: Company + name: Stanwell Corporation + aliases: [Stanwell] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: power generation, ownership: Queensland Government } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Queensland government-owned generator - Stanwell PS, Tarong, Tarong North and the Meandu mine. + + - entity_id: company:au:cleanco-qld + entity_type: Company + name: CleanCo Queensland + aliases: [CleanCo] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: power generation, ownership: Queensland Government } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Queensland's low-emissions government generator (est. 2019) - Wivenhoe pumped storage, Swanbank E, Barron Gorge, Kareeya, Karara wind. + + - entity_id: company:au:origin-energy + entity_type: Company + name: Origin Energy + aliases: [Origin] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: integrated energy, listed: true, ticker: ORG } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Gentailer - Eraring, Darling Downs, Mt Stuart, Uranquinty; 27.5% of APLNG (upstream operator). + + - entity_id: company:au:agl-energy + entity_type: Company + name: AGL Energy + aliases: [AGL] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: integrated energy, listed: true, ticker: AGL } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Australia's largest emitter and gentailer - Bayswater; operates Coopers Gap for the PARF partnership. + + - entity_id: company:au:energyaustralia + entity_type: Company + name: EnergyAustralia + aliases: [] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: integrated energy, parent: CLP Group (Hong Kong) } + external_ids: {} + sources: [source:wikidata:sparql] + notes: CLP Group subsidiary - Mt Piper, Tallawarra (and Yallourn in VIC, not modelled). + + - entity_id: company:au:delta-electricity + entity_type: Company + name: Delta Electricity + aliases: [] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: power generation, parent: Sev.en Global Investments } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Owner of Vales Point B; acquired by Czech group Sev.en Global Investments in 2022. + + - entity_id: company:au:sev-en-gi + entity_type: Company + name: Sev.en Global Investments (Australia) + aliases: [Sev.en GI] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: energy investments, parent: Sev.en Global Investments (Czech Republic) } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Czech energy investor's Australian arm - parent of Delta Electricity and, since May 2025, owner of the non-CS Energy 50% of Callide C (verified). + + - entity_id: company:au:snowy-hydro + entity_type: Company + name: Snowy Hydro + aliases: [] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: power generation, ownership: Commonwealth Government } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Commonwealth-owned hydro and gas generator - Colongra among its gas fleet; building Snowy 2.0. + + - entity_id: company:au:alinta-energy + entity_type: Company + name: Alinta Energy + aliases: [] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: integrated energy, parent: Chow Tai Fook Enterprises (Hong Kong) } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Owner of Braemar 1 (verified ex-ERM Power 2008). + + - entity_id: company:au:intergen-au + entity_type: Company + name: InterGen Australia + aliases: [InterGen] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: power generation operations } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Operator of Millmerran Power Station for the Millmerran Power Partners consortium. + + - entity_id: company:au:neoen + entity_type: Company + name: Neoen Australia + aliases: [Neoen] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: renewables IPP, parent: Brookfield } + external_ids: {} + sources: [source:wikidata:sparql] + notes: French-origin renewables developer (Western Downs GPH, Hornsdale battery); Neoen SA was acquired by Brookfield in 2025. + + - entity_id: company:au:acciona-energia + entity_type: Company + name: Acciona Energia Australia + aliases: [Acciona] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: renewables IPP, parent: Acciona Energia (Spain) } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Developer-owner of the MacIntyre wind precinct (with Ark Energy ~30%). + + - entity_id: company:au:santos + entity_type: Company + name: Santos + aliases: [] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: oil and gas, listed: true, ticker: STO } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Adelaide-based producer; GLNG operator with 30%. + + - entity_id: company:au:shell-qgc + entity_type: Company + name: Shell QGC + aliases: [QGC] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: oil and gas, parent: Shell plc } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Shell's Queensland CSG-LNG business (ex BG Group, acquired 2016); owner-operator of QCLNG. + + - entity_id: company:au:conocophillips + entity_type: Company + name: ConocoPhillips Australia + aliases: [ConocoPhillips] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: oil and gas, parent: ConocoPhillips (US) } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Holds 47.5% of APLNG (raised from 37.5% in 2022) and operates the downstream LNG facility. + + - entity_id: company:au:apa-group + entity_type: Company + name: APA Group + aliases: [APA] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: energy infrastructure, listed: true, ticker: APA } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Australia's largest gas infrastructure owner - Wallumbilla Gladstone Pipeline, Murraylink, and Basslink (since 2022). + + - entity_id: company:au:bhp-mitsubishi-alliance + entity_type: Company + name: BHP Mitsubishi Alliance + aliases: [BMA] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: metallurgical coal mining, structure: 50/50 BHP / Mitsubishi Development JV } + external_ids: {} + sources: [source:wikidata:sparql] + notes: World's largest seaborne metallurgical coal producer - Goonyella Riverside, Peak Downs, Saraji, Caval Ridge and the Hay Point Coal Terminal. Sold Blackwater and Daunia to Whitehaven in 2024. + + - entity_id: company:au:whitehaven-coal + entity_type: Company + name: Whitehaven Coal + aliases: [Whitehaven] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: coal mining, listed: true, ticker: WHC } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Acquired Blackwater and Daunia from BMA (April 2024); sold 30% of Blackwater to Nippon Steel and JFE Steel (April 2025, verified). Operates Blackwater. + + - entity_id: company:au:coronado + entity_type: Company + name: Coronado Global Resources + aliases: [Coronado] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: metallurgical coal mining, listed: true, ticker: CRN } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Owner of Curragh (plus US mines). Under liquidity pressure through 2025; Stanwell provided prepayment support tied to coal supply (verified June 2025 agreement). + + - entity_id: company:au:batchfire-resources + entity_type: Company + name: Batchfire Resources + aliases: [Batchfire] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: coal mining } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Owner of the Callide mine since October 2016 (acquired from Anglo American). + + - entity_id: company:au:glencore + entity_type: Company + name: Glencore Australia + aliases: [Glencore] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: mining and commodities trading, parent: Glencore plc } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Largest Australian thermal coal producer; 49% of Hunter Valley Operations; ~40% of WICET. + + - entity_id: company:au:anglo-american + entity_type: Company + name: Anglo American Australia + aliases: [Anglo American] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: mining, parent: Anglo American plc } + external_ids: {} + sources: [source:wikidata:sparql] + notes: >- + Former owner of the Callide mine (sold to Batchfire 2016) and of Bowen Basin steelmaking coal mines + (Moranbah North, Grosvenor, Capcoal); sale of those mines to Peabody was renegotiated during 2025 after + the Moranbah North fire - treated as in transition, so no current-mine edges are asserted in this seed. + + - entity_id: company:au:yancoal + entity_type: Company + name: Yancoal Australia + aliases: [Yancoal] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: coal mining, listed: true, ticker: YAL } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Majority Chinese-owned (Yankuang Energy); 51% owner and operator of Hunter Valley Operations; WICET shareholder. + + - entity_id: company:au:aurizon + entity_type: Company + name: Aurizon + aliases: [Aurizon Network, QR National] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: rail freight, listed: true, ticker: AZJ } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Owner-operator of the Central Queensland Coal Network (Blackwater, Moura, Goonyella systems) and a major coal haulage operator. + + - entity_id: company:au:gladstone-ports-corp + entity_type: Company + name: Gladstone Ports Corporation + aliases: [GPC] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: port authority, ownership: Queensland Government } + external_ids: {} + sources: [source:wikidata:sparql, source:gpc:cargo-statistics] + notes: Queensland government-owned port authority for Gladstone (also Rockhampton/Bundaberg); operates RG Tanna and Barney Point; publishes monthly cargo statistics. + + - entity_id: company:au:wicet-holdings + entity_type: Company + name: WICET Holdings + aliases: [Wiggins Island Coal Export Terminal Pty Ltd] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: port terminal, structure: shipper-owned consortium } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Consortium vehicle owning WICET - Glencore ~40% with Coronado Curragh and Yancoal (verified); refinanced 2018 and subsequently. + + - entity_id: company:au:dalrymple-bay-infrastructure + entity_type: Company + name: Dalrymple Bay Infrastructure + aliases: [DBI] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: port terminal, listed: true, ticker: DBI } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Holds the 99-year lease over the Queensland government-owned Dalrymple Bay Coal Terminal. + + - entity_id: company:au:tomago-aluminium + entity_type: Company + name: Tomago Aluminium Company + aliases: [TAC] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: aluminium smelting, structure: JV (Rio Tinto ~51.55%, CSR ~36.05%, Hydro ~12.4%) } + external_ids: {} + sources: [source:wikidata:sparql] + notes: JV company that owns and operates the Tomago smelter. + + - entity_id: company:au:bluescope + entity_type: Company + name: BlueScope Steel + aliases: [BlueScope] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: steel, listed: true, ticker: BSL } + external_ids: {} + sources: [source:wikidata:sparql] + notes: Owner of Port Kembla Steelworks; spun out of BHP in 2002. + + - entity_id: company:au:orica + entity_type: Company + name: Orica + aliases: [] + data_class: structural + observability: NOT_OBSERVABLE + geometry: null + country: au + grid_id: null + properties: { sector: mining chemicals and explosives, listed: true, ticker: ORI } + external_ids: {} + sources: [source:wikidata:sparql] + notes: World's largest commercial explosives supplier; owner of the Yarwun (Gladstone) ammonium nitrate complex. diff --git a/intelligence/data/entities/au-nem/gas-lng.yaml b/intelligence/data/entities/au-nem/gas-lng.yaml new file mode 100644 index 0000000..6762d53 --- /dev/null +++ b/intelligence/data/entities/au-nem/gas-lng.yaml @@ -0,0 +1,150 @@ +# Structural seed — CSG-LNG chain: Surat CSG fields, feed pipelines, Curtis Island LNG plants. +# See ../../../schemas/entity.schema.json +records: + - entity_id: gasfield:au-qld:surat-csg + entity_type: GasField + name: Surat Basin CSG Fields (aggregate) + aliases: [Surat CSG] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [150.4, -26.7] } + country: au + admin: { region: Queensland, locality: Surat Basin } + grid_id: null + properties: + resource: coal-seam gas + production_tj_day: 4300 + production_note: approximate combined CSG production feeding the three Curtis Island LNG projects plus domestic market + external_ids: {} + sources: [source:aemo:gas-bulletin-board, source:gem:global-gas-infrastructure-tracker] + notes: >- + Aggregate entity for the Surat/Bowen coal-seam gas fields (QGC's Surat fields, APLNG's Talinga/Orana/ + Combabula, GLNG's Fairview/Roma/Scotia and others). Field-level production is published daily on the + AEMO Gas Bulletin Board, making the upstream leg genuinely observable. + + - entity_id: pipe:au-qld:wallumbilla-gladstone + entity_type: Pipeline + name: Wallumbilla Gladstone Pipeline + aliases: [WGP, QCLNG Pipeline] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: LineString, coordinates: [[149.19, -26.69], [150.48, -25.30], [151.20, -23.79]] } + country: au + admin: { region: Queensland } + grid_id: null + properties: + length_km: 540 + capacity_tj_day: 1590 + capacity_note: approximate + commodity: gas + commissioned: 2014 + external_ids: {} + sources: [source:gem:global-gas-infrastructure-tracker, source:aemo:gas-bulletin-board] + notes: >- + Feed pipeline for QCLNG from the Wallumbilla hub area to Curtis Island; owned by APA Group (acquired + from BG Group in 2015). Flows reported on the AEMO Gas Bulletin Board. + + - entity_id: pipe:au-qld:aplng-pipeline + entity_type: Pipeline + name: APLNG Pipeline + aliases: [Australia Pacific LNG Pipeline] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: LineString, coordinates: [[149.50, -26.55], [150.55, -25.20], [151.21, -23.78]] } + country: au + admin: { region: Queensland } + grid_id: null + properties: + length_km: 530 + capacity_tj_day: 1560 + capacity_note: approximate + commodity: gas + commissioned: 2015 + external_ids: {} + sources: [source:gem:global-gas-infrastructure-tracker, source:aemo:gas-bulletin-board] + notes: Feed pipeline from APLNG's Surat fields (Reedy Creek/Orana area) to the APLNG facility on Curtis Island; owned by the APLNG joint venture. + + - entity_id: pipe:au-qld:glng-gtp + entity_type: Pipeline + name: GLNG Gas Transmission Pipeline + aliases: [GLNG GTP] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: LineString, coordinates: [[148.85, -25.60], [150.30, -24.60], [151.19, -23.78]] } + country: au + admin: { region: Queensland } + grid_id: null + properties: + length_km: 420 + capacity_tj_day: 1430 + capacity_note: approximate + commodity: gas + commissioned: 2015 + external_ids: {} + sources: [source:gem:global-gas-infrastructure-tracker, source:aemo:gas-bulletin-board] + notes: Feed pipeline from the Fairview/Roma CSG fields to GLNG on Curtis Island, including a Bass Strait-style marine crossing of The Narrows; owned by the GLNG joint venture (Santos-operated). + + - entity_id: lng:au-qld:qclng + entity_type: LNGPlant + name: Queensland Curtis LNG + aliases: [QCLNG] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [151.203, -23.789] } + country: au + admin: { region: Queensland, locality: Curtis Island } + grid_id: null + properties: + capacity_mtpa: 8.5 + trains: 2 + commissioned: 2014 + technology: ConocoPhillips Optimized Cascade + external_ids: {} + sources: [source:gem:global-gas-infrastructure-tracker, source:aemo:gas-bulletin-board, source:gpc:cargo-statistics] + notes: >- + First CSG-to-LNG plant in the world (first cargo January 2015). Shell-operated via QGC; CNOOC holds 50% + of Train 1 and Tokyo Gas 2.5% of Train 2. Gas deliveries visible daily via the Gas Bulletin Board; + cargo departures via GPC statistics. + + - entity_id: lng:au-qld:aplng + entity_type: LNGPlant + name: Australia Pacific LNG + aliases: [APLNG] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [151.212, -23.769] } + country: au + admin: { region: Queensland, locality: Curtis Island } + grid_id: null + properties: + capacity_mtpa: 9.0 + trains: 2 + commissioned: 2015 + technology: ConocoPhillips Optimized Cascade + external_ids: {} + sources: [source:gem:global-gas-infrastructure-tracker, source:aemo:gas-bulletin-board, source:gpc:cargo-statistics] + notes: >- + Largest of the three Curtis Island projects. JV: ConocoPhillips 47.5% (downstream operator), + Origin Energy 27.5% (upstream operator), Sinopec 25%. Sinopec is also the anchor offtaker. + + - entity_id: lng:au-qld:glng + entity_type: LNGPlant + name: Gladstone LNG + aliases: [GLNG] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [151.196, -23.779] } + country: au + admin: { region: Queensland, locality: Curtis Island } + grid_id: null + properties: + capacity_mtpa: 7.8 + trains: 2 + commissioned: 2015 + technology: ConocoPhillips Optimized Cascade + external_ids: {} + sources: [source:gem:global-gas-infrastructure-tracker, source:aemo:gas-bulletin-board, source:gpc:cargo-statistics] + notes: >- + Santos-operated JV: Santos 30%, PETRONAS 27.5%, TotalEnergies 27.5%, KOGAS 15%. Anchor offtakers are + PETRONAS (Malaysia) and KOGAS (Korea). Historically the most reliant of the three on third-party and + portfolio gas. diff --git a/intelligence/data/entities/au-nem/gladstone-cluster.yaml b/intelligence/data/entities/au-nem/gladstone-cluster.yaml new file mode 100644 index 0000000..8e179de --- /dev/null +++ b/intelligence/data/entities/au-nem/gladstone-cluster.yaml @@ -0,0 +1,155 @@ +# Structural seed — Gladstone industrial cluster (depth exemplar) plus NSW industrial-load counterparts +# (Tomago, Port Kembla, Sydney DC cluster). See ../../../schemas/entity.schema.json +records: + - entity_id: smelter:au-qld:boyne-island + entity_type: Smelter + name: Boyne Island Aluminium Smelter + aliases: [Boyne Smelters, BSL] + data_class: structural + observability: INSTRUMENTATION_OPPORTUNITY + geometry: { type: Point, coordinates: [151.355, -23.946] } + country: au + admin: { region: Queensland, locality: Boyne Island } + grid_id: grid:au-nem + properties: + product: aluminium + capacity_tpa: 500000 + potlines: 3 + technology: prebake + commissioned: 1982 + load_mw: 950 + load_note: approximately 950 MW continuous electrical load, the largest single load in Queensland + external_ids: {} + sources: [source:wikidata:sparql, source:gpc:cargo-statistics] + notes: >- + Boyne Smelters Limited JV, Rio Tinto ~73.5% and operator. Output curtailed (~14%) since 2017 due to + electricity costs. Power supplied predominantly by Gladstone Power Station under contracts running to + March 2029. No public real-time telemetry - a strong instrumentation opportunity via grid-side inference. + + - entity_id: refinery:au-qld:qal + entity_type: Refinery + name: Queensland Alumina Limited Refinery + aliases: [QAL] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [151.278, -23.860] } + country: au + admin: { region: Queensland, locality: Gladstone } + grid_id: grid:au-nem + properties: + product: alumina + capacity_tpa: 3950000 + technology: Bayer process + commissioned: 1967 + external_ids: {} + sources: [source:wikidata:sparql, source:gpc:cargo-statistics] + notes: >- + One of the world's largest alumina refineries. Rio Tinto 80%, Rusal 20%; Rio Tinto assumed 100% + operational control in 2022 with the Rusal interest quarantined under Australian sanctions. Bauxite + shipped from Weipa; alumina feeds Boyne Island and export markets. Production reported quarterly. + + - entity_id: refinery:au-qld:yarwun + entity_type: Refinery + name: Yarwun Alumina Refinery + aliases: [Rio Tinto Yarwun, Comalco Alumina Refinery] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [151.156, -23.795] } + country: au + admin: { region: Queensland, locality: Yarwun } + grid_id: grid:au-nem + properties: + product: alumina + capacity_tpa: 3000000 + technology: Bayer process + commissioned: 2004 + external_ids: {} + sources: [source:wikidata:sparql, source:gpc:cargo-statistics] + notes: Rio Tinto 100%. Expanded 2012; has its own gas-fired cogeneration. Weipa bauxite feed; alumina largely exported. + + - entity_id: chem:au-qld:orica-yarwun + entity_type: ChemicalPlant + name: Orica Yarwun Chemical Complex + aliases: [Orica Gladstone] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [151.163, -23.808] } + country: au + admin: { region: Queensland, locality: Yarwun } + grid_id: grid:au-nem + properties: + products: [ammonium nitrate, ammonia, sodium cyanide] + capacity_tpa: 680000 + capacity_note: approximately 680 ktpa ammonium nitrate; also a major sodium cyanide producer + commissioned: 1990 + external_ids: {} + sources: [source:wikidata:sparql, source:gpc:cargo-statistics] + notes: >- + Orica's largest manufacturing site; supplies bulk explosives precursors to Bowen Basin coal mining and + cyanide to gold mining. Significant natural gas and electricity consumer in the Gladstone cluster. + + - entity_id: smelter:au-nsw:tomago + entity_type: Smelter + name: Tomago Aluminium Smelter + aliases: [Tomago] + data_class: structural + observability: INSTRUMENTATION_OPPORTUNITY + geometry: { type: Point, coordinates: [151.712, -32.836] } + country: au + admin: { region: New South Wales, locality: Tomago } + grid_id: grid:au-nem + properties: + product: aluminium + capacity_tpa: 590000 + potlines: 3 + technology: prebake + commissioned: 1983 + load_mw: 950 + load_note: roughly 950 MW, around 10% of NSW electricity demand; Australia's largest single electricity consumer + external_ids: {} + sources: [source:wikidata:sparql] + notes: >- + Operated by Tomago Aluminium Company JV (Rio Tinto ~51.55%, CSR ~36.05%, Hydro Aluminium ~12.4%). + Demand-response capable (potline interruption). NSW counterpart to Boyne Island in the demand model. + + - entity_id: steel:au-nsw:port-kembla + entity_type: SteelMill + name: Port Kembla Steelworks + aliases: [BlueScope Port Kembla] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [150.893, -34.463] } + country: au + admin: { region: New South Wales, locality: Port Kembla } + grid_id: grid:au-nem + properties: + product: crude steel + capacity_tpa: 3200000 + technology: integrated blast furnace / BOS + commissioned: 1928 + external_ids: {} + sources: [source:wikidata:sparql] + notes: >- + Australia's largest steelworks, BlueScope-owned. Consumes metallurgical coal (largely from nearby + Illawarra mines) and iron ore; No. 6 blast furnace reline underway to extend life. Production + reported half-yearly. + + - entity_id: dc:au-nsw:sydney-metro-cluster + entity_type: DataCentre + name: Sydney Metro Data Centre Cluster + aliases: [Sydney DC cluster] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [151.19, -33.87] } + country: au + admin: { region: New South Wales, locality: Sydney } + grid_id: grid:au-nem + properties: + facility_count: 85 + it_load_mw: 1100 + load_note: aggregate entity; approximately 1 GW+ of commissioned IT load across the metro area, growing rapidly + external_ids: {} + sources: [source:wikidata:sparql] + notes: >- + Aggregate stand-in for Sydney's data-centre fleet (NextDC, AirTrunk, Equinix, CDC and others) - the + fastest-growing large load in NSW1. Individual sites not modelled in this structural seed. diff --git a/intelligence/data/entities/au-nem/interconnectors.yaml b/intelligence/data/entities/au-nem/interconnectors.yaml new file mode 100644 index 0000000..962bc4e --- /dev/null +++ b/intelligence/data/entities/au-nem/interconnectors.yaml @@ -0,0 +1,142 @@ +# Structural seed — NEM interconnectors. See ../../../schemas/entity.schema.json +# capacity_mw_forward is in the from_region -> to_region direction. Static nominal limits; +# actual AEMO transfer limits are dynamic and frequently lower. +records: + - entity_id: intercon:au-nem:qni + entity_type: Interconnector + name: Queensland-NSW Interconnector + aliases: [QNI] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [150.9, -28.9] } + country: au + admin: { region: Queensland/New South Wales, locality: Bulli Creek - Dumaresq } + grid_id: grid:au-nem + properties: + from_region: NSW1 + to_region: QLD1 + capacity_mw_forward: 600 + capacity_mw_reverse: 1078 + hvdc: false + commissioned: 2001 + external_ids: + aemo_duid: [NSW1-QLD1] + sources: [source:aemo:nem-registration-and-exemption-list, source:aemo:vis-nem-summary] + notes: >- + Double-circuit 330 kV AC link. Nominal limits ~600 MW northward (NSW->QLD) and ~1,078 MW southward; a + 2021-22 minor upgrade added roughly 190 MW of southward capability under favourable conditions. + + - entity_id: intercon:au-nem:terranora + entity_type: Interconnector + name: Terranora Interconnector + aliases: [Directlink, N-Q-MNSP1] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [153.50, -28.25] } + country: au + admin: { region: Queensland/New South Wales, locality: Mullumbimby - Terranora } + grid_id: grid:au-nem + properties: + from_region: NSW1 + to_region: QLD1 + capacity_mw_forward: 180 + capacity_mw_reverse: 180 + hvdc: true + commissioned: 2000 + external_ids: + aemo_duid: [N-Q-MNSP1] + sources: [source:aemo:nem-registration-and-exemption-list, source:aemo:vis-nem-summary] + notes: 59 km underground HVDC light link (3 x 60 MW); originally the merchant Directlink, converted to a regulated asset in 2006. Transfer typically limited below nameplate. + + - entity_id: intercon:au-nem:vni + entity_type: Interconnector + name: Victoria-NSW Interconnector + aliases: [VNI, VIC1-NSW1] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [146.9, -36.1] } + country: au + admin: { region: Victoria/New South Wales, locality: Murray - Wodonga corridor } + grid_id: grid:au-nem + properties: + from_region: VIC1 + to_region: NSW1 + capacity_mw_forward: 1600 + capacity_mw_reverse: 1350 + hvdc: false + commissioned: 1959 + external_ids: + aemo_duid: [VIC1-NSW1] + sources: [source:aemo:nem-registration-and-exemption-list, source:aemo:vis-nem-summary] + notes: Meshed 330/220 kV AC corridor through the Snowy region; limits are strongly condition-dependent (indicative ~1,600 MW north, ~1,350 MW south). VNI West HVDC augmentation is under development. + + - entity_id: intercon:au-nem:murraylink + entity_type: Interconnector + name: Murraylink + aliases: [V-MNSP1] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [141.5, -34.3] } + country: au + admin: { region: Victoria/South Australia, locality: Red Cliffs - Berri } + grid_id: grid:au-nem + properties: + from_region: VIC1 + to_region: SA1 + capacity_mw_forward: 220 + capacity_mw_reverse: 200 + hvdc: true + commissioned: 2002 + external_ids: + aemo_duid: [V-MNSP1] + sources: [source:aemo:nem-registration-and-exemption-list, source:aemo:vis-nem-summary] + notes: 180 km underground HVDC light cable, the world's longest when built; regulated asset owned by APA Group. + + - entity_id: intercon:au-nem:heywood + entity_type: Interconnector + name: Heywood Interconnector + aliases: [V-SA, VIC1-SA1] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [141.3, -38.1] } + country: au + admin: { region: Victoria/South Australia, locality: Heywood - South East } + grid_id: grid:au-nem + properties: + from_region: VIC1 + to_region: SA1 + capacity_mw_forward: 650 + capacity_mw_reverse: 650 + hvdc: false + commissioned: 1988 + external_ids: + aemo_duid: [V-SA] + sources: [source:aemo:nem-registration-and-exemption-list, source:aemo:vis-nem-summary] + notes: 275 kV AC link, SA's main interconnection; upgraded to ~650 MW nominal each way in 2016. Frequently constrained below nominal for system security. + + - entity_id: intercon:au-nem:basslink + entity_type: Interconnector + name: Basslink + aliases: [T-V-MNSP1] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [146.7, -39.7] } + country: au + admin: { region: Tasmania/Victoria, locality: George Town - Loy Yang } + grid_id: grid:au-nem + properties: + from_region: TAS1 + to_region: VIC1 + capacity_mw_forward: 500 + capacity_mw_reverse: 478 + hvdc: true + commissioned: 2006 + regulated_from: "2026-07" + external_ids: + aemo_duid: [T-V-MNSP1] + sources: [source:aemo:nem-registration-and-exemption-list, source:aemo:vis-nem-summary] + notes: >- + 370 km HVDC monopole across Bass Strait (500 MW continuous, up to ~630 MW short-term TAS->VIC). + Owned by APA Group since 2022 (acquired out of the Basslink Pty Ltd administration) - not state-owned. + The AER approved conversion from a market network service to a prescribed (regulated) transmission + service in June 2025, with regulated operation from July 2026 (verified). diff --git a/intelligence/data/entities/au-nem/mines-basins.yaml b/intelligence/data/entities/au-nem/mines-basins.yaml new file mode 100644 index 0000000..fc55f44 --- /dev/null +++ b/intelligence/data/entities/au-nem/mines-basins.yaml @@ -0,0 +1,219 @@ +# Structural seed — coal mines and resource basins. See ../../../schemas/entity.schema.json +# Production figures are approximate recent-year rates (Mtpa saleable unless noted). +records: + - entity_id: mine:au-qld:blackwater + entity_type: CoalMine + name: Blackwater Mine + aliases: [Blackwater] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [148.88, -23.70] } + country: au + admin: { region: Queensland, locality: Blackwater } + grid_id: null + properties: + mine_type: open-cut + commodities: [metallurgical coal, thermal coal] + production_mtpa: 13 + production_note: approximate; one of Australia's largest open-cut coal complexes (~80 km strike) + external_ids: {} + sources: [source:gem:global-coal-mine-tracker, source:wikidata:sparql] + notes: >- + Sold by BHP Mitsubishi Alliance to Whitehaven Coal (completed April 2024); Whitehaven sold 30% to + Nippon Steel (20%) and JFE Steel (10%) effective April 2025 (verified). Whitehaven operates. Exports + via the Blackwater rail system to Gladstone. + + - entity_id: mine:au-qld:goonyella-riverside + entity_type: CoalMine + name: Goonyella Riverside Mine + aliases: [Goonyella] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [147.976, -21.746] } + country: au + admin: { region: Queensland, locality: Moranbah } + grid_id: null + properties: + mine_type: open-cut + commodities: [metallurgical coal] + production_mtpa: 16 + production_note: approximate; flagship BMA hard-coking-coal mine + external_ids: {} + sources: [source:gem:global-coal-mine-tracker, source:wikidata:sparql] + notes: BHP Mitsubishi Alliance (50/50); exports via the Goonyella rail system to Hay Point. + + - entity_id: mine:au-qld:peak-downs + entity_type: CoalMine + name: Peak Downs Mine + aliases: [Peak Downs] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [148.19, -22.26] } + country: au + admin: { region: Queensland, locality: Moranbah } + grid_id: null + properties: + mine_type: open-cut + commodities: [metallurgical coal] + production_mtpa: 11 + production_note: approximate + external_ids: {} + sources: [source:gem:global-coal-mine-tracker, source:wikidata:sparql] + notes: BHP Mitsubishi Alliance (50/50) hard coking coal; exports via the Goonyella rail system to Hay Point. + + - entity_id: mine:au-qld:curragh + entity_type: CoalMine + name: Curragh Mine + aliases: [Curragh] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [148.83, -23.45] } + country: au + admin: { region: Queensland, locality: Blackwater } + grid_id: null + properties: + mine_type: open-cut + commodities: [metallurgical coal, thermal coal] + production_mtpa: 12 + production_note: approximate; ~10 Mtpa export metallurgical plus ~2-3 Mtpa domestic thermal to Stanwell + external_ids: {} + sources: [source:gem:global-coal-mine-tracker, source:wikidata:sparql] + notes: >- + Coronado Global Resources. Supplies thermal coal to Stanwell Power Station under public contract + (legacy agreement to early 2027, replacement agreements from 2027 including a June 2025 deal - verified). + Exports via the Blackwater rail system to RG Tanna and WICET. + + - entity_id: mine:au-qld:callide + entity_type: CoalMine + name: Callide Mine + aliases: [Callide] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [150.63, -24.30] } + country: au + admin: { region: Queensland, locality: Biloela } + grid_id: null + properties: + mine_type: open-cut + commodities: [thermal coal] + production_mtpa: 8 + production_note: approximate capacity ~10 Mtpa; substantially a captive supplier to the adjacent Callide B and C stations + external_ids: {} + sources: [source:gem:global-coal-mine-tracker, source:wikidata:sparql] + notes: >- + Batchfire Resources since October 2016 (previously Anglo American). Geologically in the small Callide + Basin rather than the Bowen Basin proper. Surplus coal exported through Gladstone via the Moura rail system. + + - entity_id: mine:au-qld:meandu + entity_type: CoalMine + name: Meandu Mine + aliases: [Meandu] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [151.85, -26.80] } + country: au + admin: { region: Queensland, locality: Nanango } + grid_id: null + properties: + mine_type: open-cut + commodities: [thermal coal] + production_mtpa: 7 + production_note: approximate; captive supplier + external_ids: {} + sources: [source:gem:global-coal-mine-tracker, source:wikidata:sparql] + notes: Owned by Stanwell Corporation; dedicated fuel source for Tarong and Tarong North (conveyor/short haul). In the Tarong Basin. + + - entity_id: mine:au-qld:kogan-creek + entity_type: CoalMine + name: Kogan Creek Mine + aliases: [Kogan Creek Coal Mine] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [150.76, -26.95] } + country: au + admin: { region: Queensland, locality: Chinchilla } + grid_id: null + properties: + mine_type: open-cut + commodities: [thermal coal] + production_mtpa: 2.8 + production_note: approximate; captive supplier + external_ids: {} + sources: [source:gem:global-coal-mine-tracker] + notes: CS Energy-owned captive mine feeding Kogan Creek Power Station by conveyor. Walloon coal measures, Surat Basin. + + - entity_id: mine:au-nsw:hunter-valley-operations + entity_type: CoalMine + name: Hunter Valley Operations + aliases: [HVO] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [151.06, -32.52] } + country: au + admin: { region: New South Wales, locality: Singleton } + grid_id: null + properties: + mine_type: open-cut + commodities: [thermal coal, semi-soft coking coal] + production_mtpa: 13 + production_note: approximate saleable + external_ids: {} + sources: [source:gem:global-coal-mine-tracker, source:wikidata:sparql] + notes: Yancoal (51%, operator) / Glencore (49%) JV in the Hunter Valley (Sydney Basin); exports via the Port of Newcastle. + + - entity_id: basin:au:bowen + entity_type: CoalBasin + name: Bowen Basin + aliases: [] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [148.0, -22.5] } + country: au + admin: { region: Queensland } + grid_id: null + properties: + area_km2: 60000 + primary_commodity: metallurgical coal + external_ids: {} + sources: [source:gem:global-coal-mine-tracker, source:wikidata:sparql] + notes: >- + The world's premier seaborne metallurgical coal province, central Queensland; also hosts coal-seam gas + (Fairview/Scotia etc. feeding GLNG). Aggregate production visible via Queensland government and GEM data. + + - entity_id: basin:au:surat + entity_type: GasBasin + name: Surat Basin + aliases: [] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [150.0, -27.0] } + country: au + admin: { region: Queensland } + grid_id: null + properties: + area_km2: 270000 + primary_commodity: coal-seam gas + external_ids: {} + sources: [source:gem:global-gas-infrastructure-tracker, source:aemo:gas-bulletin-board] + notes: >- + Principal coal-seam gas province feeding the Curtis Island LNG projects; also hosts Walloon-measure + thermal coal (Kogan Creek, Cameby Downs). Gas production reported daily via the AEMO Gas Bulletin Board. + + - entity_id: basin:au:galilee + entity_type: CoalBasin + name: Galilee Basin + aliases: [] + data_class: structural + observability: NOT_OBSERVABLE + geometry: { type: Point, coordinates: [145.8, -22.8] } + country: au + admin: { region: Queensland } + grid_id: null + properties: + area_km2: 247000 + primary_commodity: thermal coal + external_ids: {} + sources: [source:gem:global-coal-mine-tracker, source:wikidata:sparql] + notes: >- + Largely undeveloped inland thermal coal basin; the Bravus (Adani) Carmichael mine (exporting via Abbot + Point) is its only operating project and is not modelled in this seed. Included as strategic context. diff --git a/intelligence/data/entities/au-nem/nsw-generators.yaml b/intelligence/data/entities/au-nem/nsw-generators.yaml new file mode 100644 index 0000000..d469d8b --- /dev/null +++ b/intelligence/data/entities/au-nem/nsw-generators.yaml @@ -0,0 +1,171 @@ +# Structural seed — New South Wales generators (NSW1). See ../../../schemas/entity.schema.json +records: + - entity_id: gen:au-nem:eraring + entity_type: Generator + name: Eraring Power Station + aliases: [Eraring] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [151.521, -33.062] } + country: au + admin: { region: New South Wales, locality: Eraring } + grid_id: grid:au-nem + properties: + fuel: coal + capacity_mw: 2880 + technology: steam subcritical + commissioned: 1982 + unit_count: 4 + status: operating + closure_announced: "2029-04-30" + external_ids: + aemo_duid: [ER01, ER02, ER03, ER04] + aemo_station_id: ERARING + sources: [source:aemo:nem-registration-and-exemption-list, source:gem:global-coal-plant-tracker] + notes: >- + Australia's largest power station; Origin Energy. Closure was announced for August 2025, extended to August + 2027 under a May 2024 underwriting agreement with the NSW Government, then extended again in February 2026 + to 30 April 2029 (verified Feb 2026 Origin announcement). Co-located Eraring battery (700 MW / 3,160 MWh + when complete) is a separate asset. + + - entity_id: gen:au-nem:bayswater + entity_type: Generator + name: Bayswater Power Station + aliases: [Bayswater] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [150.948, -32.393] } + country: au + admin: { region: New South Wales, locality: Muswellbrook } + grid_id: grid:au-nem + properties: + fuel: coal + capacity_mw: 2640 + technology: steam subcritical + commissioned: 1985 + unit_count: 4 + status: operating + closure_announced: "2033" + external_ids: + aemo_duid: [BW01, BW02, BW03, BW04] + aemo_station_id: BAYSW + sources: [source:aemo:nem-registration-and-exemption-list, source:gem:global-coal-plant-tracker] + notes: AGL Energy, Hunter Valley; AGL has announced closure no later than 2033. Coal from Hunter Valley mines by rail and conveyor. + + - entity_id: gen:au-nem:mt-piper + entity_type: Generator + name: Mt Piper Power Station + aliases: [Mount Piper] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [150.074, -33.360] } + country: au + admin: { region: New South Wales, locality: Portland } + grid_id: grid:au-nem + properties: + fuel: coal + capacity_mw: 1400 + technology: steam subcritical + commissioned: 1993 + unit_count: 2 + status: operating + external_ids: + aemo_duid: [MP1, MP2] + aemo_station_id: MTPIPER + sources: [source:aemo:nem-registration-and-exemption-list, source:gem:global-coal-plant-tracker] + notes: EnergyAustralia; western coalfields near Lithgow, supplied mainly by Centennial's Springvale/Angus Place complex (not modelled in this seed). Expected to be NSW's last operating coal station (~2040). + + - entity_id: gen:au-nem:vales-point + entity_type: Generator + name: Vales Point B Power Station + aliases: [Vales Point] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [151.526, -33.163] } + country: au + admin: { region: New South Wales, locality: Mannering Park } + grid_id: grid:au-nem + properties: + fuel: coal + capacity_mw: 1320 + technology: steam subcritical + commissioned: 1978 + unit_count: 2 + status: operating + closure_announced: "2033" + external_ids: + aemo_duid: [VP5, VP6] + aemo_station_id: VPGS + sources: [source:aemo:nem-registration-and-exemption-list, source:gem:global-coal-plant-tracker] + notes: >- + Owned by Delta Electricity (Sev.en Global Investments group since 2022). Closure previously flagged for 2029, + now indicated around 2033; confidence moderate as timing remains commercially contingent. + + - entity_id: gen:au-nem:tallawarra + entity_type: Generator + name: Tallawarra Power Station + aliases: [Tallawarra A, Tallawarra B] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [150.809, -34.535] } + country: au + admin: { region: New South Wales, locality: Yallah } + grid_id: grid:au-nem + properties: + fuel: gas + capacity_mw: 755 + technology: combined-cycle (A, 435 MW) plus open-cycle (B, 320 MW) + commissioned: 2009 + unit_count: 2 + status: operating + external_ids: + aemo_duid: [TALWA1, TALWB1] + aemo_station_id: TALWA + sources: [source:aemo:nem-registration-and-exemption-list] + notes: EnergyAustralia, Lake Illawarra. Tallawarra B (2024) is a dual-fuel hydrogen-capable OCGT; modelled here as one station record covering both stages. + + - entity_id: gen:au-nem:colongra + entity_type: Generator + name: Colongra Power Station + aliases: [Colongra] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [151.543, -33.205] } + country: au + admin: { region: New South Wales, locality: Colongra } + grid_id: grid:au-nem + properties: + fuel: gas + capacity_mw: 667 + technology: open-cycle gas turbine + commissioned: 2009 + unit_count: 4 + status: operating + external_ids: + aemo_duid: [CG1, CG2, CG3, CG4] + aemo_station_id: CG + sources: [source:aemo:nem-registration-and-exemption-list] + notes: Snowy Hydro peaker on the Central Coast, adjacent to the former Munmorah site. + + - entity_id: gen:au-nem:uranquinty + entity_type: Generator + name: Uranquinty Power Station + aliases: [Uranquinty] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [147.247, -35.194] } + country: au + admin: { region: New South Wales, locality: Uranquinty } + grid_id: grid:au-nem + properties: + fuel: gas + capacity_mw: 664 + technology: open-cycle gas turbine + commissioned: 2009 + unit_count: 4 + status: operating + external_ids: + aemo_duid: [URANQ11, URANQ12, URANQ13, URANQ14] + aemo_station_id: URANQ + sources: [source:aemo:nem-registration-and-exemption-list] + notes: Origin Energy peaker near Wagga Wagga. diff --git a/intelligence/data/entities/au-nem/ports-terminals-rail.yaml b/intelligence/data/entities/au-nem/ports-terminals-rail.yaml new file mode 100644 index 0000000..287e4e6 --- /dev/null +++ b/intelligence/data/entities/au-nem/ports-terminals-rail.yaml @@ -0,0 +1,214 @@ +# Structural seed — export ports, terminals and coal rail systems. See ../../../schemas/entity.schema.json +records: + - entity_id: port:au-qld:gladstone + entity_type: Port + name: Port of Gladstone + aliases: [Gladstone Harbour] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [151.25, -23.83] } + country: au + admin: { region: Queensland, locality: Gladstone } + grid_id: null + properties: + harbour_type: natural deepwater + max_draught_m: 18 + throughput_mtpa: 120 + throughput_note: approximately 120 Mtpa across coal, LNG, alumina, aluminium, bauxite and cement + external_ids: + unlocode: AUGLT + sources: [source:gpc:cargo-statistics, source:wikidata:sparql] + notes: >- + Queensland's largest multi-commodity port and one of the world's top coal and LNG export harbours; + managed by Gladstone Ports Corporation, which publishes monthly cargo statistics (the key public + observability feed for the cluster). Includes the Curtis Island LNG jetties. + + - entity_id: port:au-nsw:newcastle + entity_type: Port + name: Port of Newcastle + aliases: [Newcastle Harbour] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [151.786, -32.918] } + country: au + admin: { region: New South Wales, locality: Newcastle } + grid_id: null + properties: + harbour_type: river harbour + max_draught_m: 16.5 + throughput_mtpa: 160 + throughput_note: approximate; the world's largest coal export port (PWCS and NCIG terminals, not modelled separately) + external_ids: + unlocode: AUNTL + sources: [source:wikidata:sparql] + notes: Hunter Valley coal chain outlet; privatised under long-term lease in 2014. + + - entity_id: port:au-qld:hay-point + entity_type: Port + name: Port of Hay Point + aliases: [Hay Point] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [149.30, -21.28] } + country: au + admin: { region: Queensland, locality: Hay Point } + grid_id: null + properties: + harbour_type: offshore jetty + max_draught_m: 14.5 + throughput_mtpa: 110 + throughput_note: approximate combined HPCT + DBCT metallurgical coal exports + external_ids: + unlocode: AUHPT + sources: [source:wikidata:sparql] + notes: >- + Twin-terminal metallurgical coal port south of Mackay serving the Goonyella rail system - NOT part of + the Port of Gladstone. Managed by North Queensland Bulk Ports (state-owned). + + - entity_id: terminal:au-qld:rg-tanna + entity_type: Terminal + name: RG Tanna Coal Terminal + aliases: [RGTCT] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [151.220, -23.842] } + country: au + admin: { region: Queensland, locality: Gladstone } + grid_id: null + properties: + commodities: [coal] + nameplate_mtpa: 40 + nameplate_note: approximately 40 Mtpa + external_ids: {} + sources: [source:gpc:cargo-statistics, source:wikidata:sparql] + notes: Gladstone Ports Corporation's main coal terminal (four berths); served by the Blackwater and Moura rail systems. + + - entity_id: terminal:au-qld:wicet + entity_type: Terminal + name: Wiggins Island Coal Export Terminal + aliases: [WICET] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [151.19, -23.81] } + country: au + admin: { region: Queensland, locality: Gladstone } + grid_id: null + properties: + commodities: [coal] + nameplate_mtpa: 27 + external_ids: {} + sources: [source:gpc:cargo-statistics, source:wikidata:sparql] + notes: >- + Independent user-funded terminal (first shipment 2015), 27 Mtpa stage one (verified). Owned by its + shipper consortium - Glencore ~40% with Coronado Curragh and Yancoal - not by Gladstone Ports Corporation. + + - entity_id: terminal:au-qld:barney-point + entity_type: Terminal + name: Barney Point Terminal + aliases: [Barney Point] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [151.273, -23.850] } + country: au + admin: { region: Queensland, locality: Gladstone } + grid_id: null + properties: + commodities: [alumina, petroleum coke, gypsum, general bulk] + nameplate_mtpa: 5 + nameplate_note: approximate + external_ids: {} + sources: [source:gpc:cargo-statistics] + notes: Gladstone Ports Corporation berth adjacent to town; coal exports ceased in 2016, now handles non-coal dry bulk. + + - entity_id: terminal:au-qld:hay-point + entity_type: Terminal + name: Hay Point Coal Terminal + aliases: [HPCT] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [149.304, -21.276] } + country: au + admin: { region: Queensland, locality: Hay Point } + grid_id: null + properties: + commodities: [coal] + nameplate_mtpa: 55 + external_ids: {} + sources: [source:wikidata:sparql] + notes: BHP Mitsubishi Alliance's dedicated export terminal for its Goonyella-system mines (Goonyella Riverside, Peak Downs, Saraji, Caval Ridge). + + - entity_id: terminal:au-qld:dbct + entity_type: Terminal + name: Dalrymple Bay Coal Terminal + aliases: [DBCT] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [149.288, -21.263] } + country: au + admin: { region: Queensland, locality: Hay Point } + grid_id: null + properties: + commodities: [coal] + nameplate_mtpa: 85 + external_ids: {} + sources: [source:wikidata:sparql] + notes: >- + World's largest single metallurgical coal export terminal; Queensland government-owned, operated under + 99-year lease by Dalrymple Bay Infrastructure. Common-user terminal for non-BMA Goonyella-system miners. + + - entity_id: rail:au-qld:blackwater-system + entity_type: Railway + name: Blackwater Rail System + aliases: [] + data_class: structural + observability: ESTIMATED + geometry: { type: LineString, coordinates: [[151.22, -23.84], [150.33, -23.51], [148.88, -23.59]] } + country: au + admin: { region: Queensland } + grid_id: null + properties: + capacity_mtpa: 70 + capacity_note: approximate contracted system capacity + electrified: true + gauge_mm: 1067 + external_ids: {} + sources: [source:wikidata:sparql] + notes: Central Queensland Coal Network corridor linking Bowen Basin mines (Blackwater, Curragh and others) to RG Tanna and WICET at Gladstone; owned and operated by Aurizon Network. + + - entity_id: rail:au-qld:moura-system + entity_type: Railway + name: Moura Rail System + aliases: [] + data_class: structural + observability: ESTIMATED + geometry: { type: LineString, coordinates: [[151.22, -23.84], [150.63, -24.30], [149.97, -24.57]] } + country: au + admin: { region: Queensland } + grid_id: null + properties: + capacity_mtpa: 13 + capacity_note: approximate + electrified: false + gauge_mm: 1067 + external_ids: {} + sources: [source:wikidata:sparql] + notes: Non-electrified CQCN corridor linking Moura and Callide/Boundary Hill mines to Gladstone; Aurizon Network. + + - entity_id: rail:au-qld:goonyella-system + entity_type: Railway + name: Goonyella Rail System + aliases: [] + data_class: structural + observability: ESTIMATED + geometry: { type: LineString, coordinates: [[149.29, -21.27], [148.19, -22.26], [147.98, -21.75]] } + country: au + admin: { region: Queensland } + grid_id: null + properties: + capacity_mtpa: 140 + capacity_note: approximate contracted system capacity + electrified: true + gauge_mm: 1067 + external_ids: {} + sources: [source:wikidata:sparql] + notes: Highest-volume CQCN corridor linking northern Bowen Basin metallurgical mines to Hay Point (HPCT and DBCT); owned and operated by Aurizon Network. diff --git a/intelligence/data/entities/au-nem/qld-generators.yaml b/intelligence/data/entities/au-nem/qld-generators.yaml new file mode 100644 index 0000000..2200d4c --- /dev/null +++ b/intelligence/data/entities/au-nem/qld-generators.yaml @@ -0,0 +1,443 @@ +# Structural seed — Queensland generators (QLD1). See ../../../schemas/entity.schema.json +records: + - entity_id: gen:au-nem:gladstone-ps + entity_type: Generator + name: Gladstone Power Station + aliases: [GPS] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [151.171, -23.855] } + country: au + admin: { region: Queensland, locality: Gladstone } + grid_id: grid:au-nem + properties: + fuel: coal + capacity_mw: 1680 + technology: steam subcritical + commissioned: 1976 + unit_count: 6 + status: operating + closure_announced: "2029-03" + external_ids: + aemo_duid: [GSTONE1, GSTONE2, GSTONE3, GSTONE4, GSTONE5, GSTONE6] + aemo_station_id: GSTONE + sources: [source:aemo:nem-registration-and-exemption-list, source:gem:global-coal-plant-tracker] + notes: >- + Queensland's largest power station; NRG operates for the Gladstone JV (Rio Tinto 42.125%, NRG Energy 37.5%, + Southern Cross GPS 8.25%, Ryowa II GPS 7.125%, YKK GPS 4.75%). Most output is committed to the Boyne Island + smelter. On 1 October 2025 CS Energy (as market intermediary) formally notified AEMO of potential retirement + in March 2029, aligned with expiry of supply and operating arrangements incl. the Boyne Smelters contract; + no final closure decision has been made (verified Oct 2025 Rio Tinto release). + + - entity_id: gen:au-nem:callide-b + entity_type: Generator + name: Callide B Power Station + aliases: [Callide B] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [150.627, -24.343] } + country: au + admin: { region: Queensland, locality: Biloela } + grid_id: grid:au-nem + properties: + fuel: coal + capacity_mw: 700 + technology: steam subcritical + commissioned: 1988 + unit_count: 2 + status: operating + external_ids: + aemo_duid: [CALL_B_1, CALL_B_2] + aemo_station_id: CALL_B + sources: [source:aemo:nem-registration-and-exemption-list, source:gem:global-coal-plant-tracker] + notes: >- + CS Energy owned and operated; fed by the adjacent Callide mine. Closure around 2028 was flagged under the + 2022 Queensland Energy and Jobs Plan; that timeline is under review by the current Queensland government, + so no firm retirement date is recorded here. + + - entity_id: gen:au-nem:callide-c + entity_type: Generator + name: Callide C Power Station + aliases: [Callide C] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [150.632, -24.345] } + country: au + admin: { region: Queensland, locality: Biloela } + grid_id: grid:au-nem + properties: + fuel: coal + capacity_mw: 810 + technology: steam supercritical + commissioned: 2001 + unit_count: 2 + status: operating + external_ids: + aemo_duid: [CPP_3, CPP_4] + aemo_station_id: CPP + sources: [source:aemo:nem-registration-and-exemption-list, source:gem:global-coal-plant-tracker] + notes: >- + 50/50 unincorporated JV. C4 was destroyed by a turbine hall explosion in May 2021; both units returned to + service across 2023-24. The non-CS Energy 50% (formerly IG Power / Genuity, in administration from March 2023) + was acquired and recapitalised by Sev.en Global Investments in May 2025 (verified). CS Energy operates. + Unit ratings quoted variously 405-420 MW each; 810 MW nameplate used here. + + - entity_id: gen:au-nem:stanwell + entity_type: Generator + name: Stanwell Power Station + aliases: [Stanwell PS] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [150.331, -23.508] } + country: au + admin: { region: Queensland, locality: Stanwell } + grid_id: grid:au-nem + properties: + fuel: coal + capacity_mw: 1460 + technology: steam subcritical + commissioned: 1993 + unit_count: 4 + status: operating + external_ids: + aemo_duid: [STAN-1, STAN-2, STAN-3, STAN-4] + aemo_station_id: STANWELL + sources: [source:aemo:nem-registration-and-exemption-list, source:gem:global-coal-plant-tracker] + notes: >- + Stanwell Corporation's flagship station near Rockhampton; coal railed from the Curragh mine under a + long-standing supply arrangement (see fuel-supply relationships). + + - entity_id: gen:au-nem:kogan-creek + entity_type: Generator + name: Kogan Creek Power Station + aliases: [Kogan Creek] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [150.746, -26.928] } + country: au + admin: { region: Queensland, locality: Chinchilla } + grid_id: grid:au-nem + properties: + fuel: coal + capacity_mw: 744 + technology: steam supercritical, air-cooled + commissioned: 2007 + unit_count: 1 + status: operating + external_ids: + aemo_duid: [KPP_1] + aemo_station_id: KPP + sources: [source:aemo:nem-registration-and-exemption-list, source:gem:global-coal-plant-tracker] + notes: Australia's largest single generating unit; CS Energy owned, fed by the adjacent captive Kogan Creek mine. + + - entity_id: gen:au-nem:tarong + entity_type: Generator + name: Tarong Power Station + aliases: [Tarong PS] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [151.914, -26.781] } + country: au + admin: { region: Queensland, locality: Nanango } + grid_id: grid:au-nem + properties: + fuel: coal + capacity_mw: 1400 + technology: steam subcritical + commissioned: 1984 + unit_count: 4 + status: operating + external_ids: + aemo_duid: ["TARONG#1", "TARONG#2", "TARONG#3", "TARONG#4"] + aemo_station_id: TARONG + sources: [source:aemo:nem-registration-and-exemption-list, source:gem:global-coal-plant-tracker] + notes: Stanwell Corporation; fed by the captive Meandu mine. + + - entity_id: gen:au-nem:tarong-north + entity_type: Generator + name: Tarong North Power Station + aliases: [Tarong North] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [151.910, -26.777] } + country: au + admin: { region: Queensland, locality: Nanango } + grid_id: grid:au-nem + properties: + fuel: coal + capacity_mw: 443 + technology: steam supercritical + commissioned: 2003 + unit_count: 1 + status: operating + external_ids: + aemo_duid: [TNPS1] + aemo_station_id: TNPS + sources: [source:aemo:nem-registration-and-exemption-list, source:gem:global-coal-plant-tracker] + notes: Supercritical single unit adjacent to Tarong; Stanwell Corporation; Meandu coal. + + - entity_id: gen:au-nem:millmerran + entity_type: Generator + name: Millmerran Power Station + aliases: [Millmerran] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [151.271, -27.956] } + country: au + admin: { region: Queensland, locality: Millmerran } + grid_id: grid:au-nem + properties: + fuel: coal + capacity_mw: 852 + technology: steam supercritical, air-cooled + commissioned: 2002 + unit_count: 2 + status: operating + external_ids: + aemo_duid: [MPP_1, MPP_2] + aemo_station_id: MPP + sources: [source:aemo:nem-registration-and-exemption-list, source:gem:global-coal-plant-tracker] + notes: >- + Operated by InterGen for the Millmerran Power Partners consortium; fed by the adjacent captive + Commodore mine (not modelled separately in this seed). + + - entity_id: gen:au-nem:wivenhoe + entity_type: Generator + name: Wivenhoe Power Station + aliases: [Wivenhoe Pumped Storage, Splityard Creek] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [152.608, -27.394] } + country: au + admin: { region: Queensland, locality: Wivenhoe Pocket } + grid_id: grid:au-nem + properties: + fuel: pumped-hydro + capacity_mw: 570 + technology: pumped storage hydro + commissioned: 1984 + unit_count: 2 + status: operating + storage_note: Pumps from Lake Wivenhoe to Splityard Creek dam off-peak; net electricity consumer over a cycle. + external_ids: + aemo_duid: ["W/HOE#1", "W/HOE#2"] + aemo_station_id: WHOE + sources: [source:aemo:nem-registration-and-exemption-list] + notes: Queensland's only operating pumped-storage station; owned by CleanCo Queensland (transferred from CS Energy in 2019). + + - entity_id: gen:au-nem:braemar + entity_type: Generator + name: Braemar Power Station + aliases: [Braemar 1] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [150.868, -27.128] } + country: au + admin: { region: Queensland, locality: Dalby } + grid_id: grid:au-nem + properties: + fuel: gas + capacity_mw: 504 + technology: open-cycle gas turbine + commissioned: 2006 + unit_count: 3 + status: operating + external_ids: + aemo_duid: [BRAEMAR1, BRAEMAR2, BRAEMAR3] + aemo_station_id: BRAEMAR + sources: [source:aemo:nem-registration-and-exemption-list] + notes: >- + Braemar 1 (this record) is owned by Alinta Energy (acquired from ERM Power in 2008, verified). The adjacent + Braemar 2 station (BRAEMAR5-7, ~519 MW) is separately owned by Arrow Energy and is not modelled in this seed. + Coal-seam gas fuelled. + + - entity_id: gen:au-nem:darling-downs + entity_type: Generator + name: Darling Downs Power Station + aliases: [Darling Downs] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [150.837, -27.106] } + country: au + admin: { region: Queensland, locality: Dalby } + grid_id: grid:au-nem + properties: + fuel: gas + capacity_mw: 644 + technology: combined-cycle gas turbine + commissioned: 2010 + unit_count: 4 + status: operating + external_ids: + aemo_duid: [DDPS1] + aemo_station_id: DDPS + sources: [source:aemo:nem-registration-and-exemption-list] + notes: Australia's largest combined-cycle plant when built; Origin Energy; coal-seam gas fuelled. + + - entity_id: gen:au-nem:swanbank-e + entity_type: Generator + name: Swanbank E Power Station + aliases: [Swanbank E] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [152.830, -27.652] } + country: au + admin: { region: Queensland, locality: Ipswich } + grid_id: grid:au-nem + properties: + fuel: gas + capacity_mw: 385 + technology: combined-cycle gas turbine + commissioned: 2002 + unit_count: 1 + status: operating + external_ids: + aemo_duid: [SWAN_E] + aemo_station_id: SWAN_E + sources: [source:aemo:nem-registration-and-exemption-list] + notes: >- + CleanCo Queensland. Mothballed 2017-2022, returned to service October 2022; offline April-October 2025 + after a gas turbine cooling-air fault, since returned to service (verified). + + - entity_id: gen:au-nem:mt-stuart + entity_type: Generator + name: Mt Stuart Power Station + aliases: [Mount Stuart] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [146.833, -19.352] } + country: au + admin: { region: Queensland, locality: Townsville } + grid_id: grid:au-nem + properties: + fuel: diesel + capacity_mw: 423 + technology: open-cycle gas turbine (kerosene-fuelled) + commissioned: 1999 + unit_count: 3 + status: operating + external_ids: + aemo_duid: [MSTUART1, MSTUART2, MSTUART3] + aemo_station_id: MSTUART + sources: [source:aemo:nem-registration-and-exemption-list] + notes: Origin Energy peaker in Townsville; runs on aviation-grade kerosene (recorded under fuel 'diesel' as nearest liquid-fuel enum). Unit 3 added 2009. + + - entity_id: gen:au-nem:barron-gorge + entity_type: Generator + name: Barron Gorge Hydro Power Station + aliases: [Barron Gorge] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [145.648, -16.845] } + country: au + admin: { region: Queensland, locality: Kuranda } + grid_id: grid:au-nem + properties: + fuel: hydro + capacity_mw: 66 + technology: run-of-river hydro + commissioned: 1963 + unit_count: 2 + status: operating + external_ids: + aemo_duid: [BARRON-1, BARRON-2] + aemo_station_id: BARRON + sources: [source:aemo:nem-registration-and-exemption-list] + notes: Far North Queensland hydro on the Barron River; CleanCo Queensland. + + - entity_id: gen:au-nem:kareeya + entity_type: Generator + name: Kareeya Hydro Power Station + aliases: [Kareeya] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [145.577, -17.776] } + country: au + admin: { region: Queensland, locality: Tully } + grid_id: grid:au-nem + properties: + fuel: hydro + capacity_mw: 88 + technology: run-of-river hydro + commissioned: 1957 + unit_count: 4 + status: operating + external_ids: + aemo_duid: [KAREEYA1, KAREEYA2, KAREEYA3, KAREEYA4] + aemo_station_id: KAREEYA + sources: [source:aemo:nem-registration-and-exemption-list] + notes: Hydro on the Tully River, Far North Queensland; CleanCo Queensland. + + - entity_id: gen:au-nem:coopers-gap + entity_type: Generator + name: Coopers Gap Wind Farm + aliases: [Coopers Gap] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [151.418, -26.723] } + country: au + admin: { region: Queensland, locality: Cooranga North } + grid_id: grid:au-nem + properties: + fuel: wind + capacity_mw: 453 + technology: onshore wind (123 turbines) + commissioned: 2020 + unit_count: 123 + status: operating + external_ids: + aemo_duid: [COOPGWF1] + aemo_station_id: COOPGWF + sources: [source:aemo:nem-registration-and-exemption-list] + notes: Owned by the Powering Australian Renewables partnership (AGL 20% with QIC/Future Fund); operated by AGL. + + - entity_id: gen:au-nem:western-downs-gph + entity_type: Generator + name: Western Downs Green Power Hub + aliases: [Western Downs Solar] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [150.79, -26.88] } + country: au + admin: { region: Queensland, locality: Chinchilla } + grid_id: grid:au-nem + properties: + fuel: solar + capacity_mw: 460 + technology: single-axis tracking PV (460 MWp DC, ~400 MW AC) + commissioned: 2023 + unit_count: 1 + status: operating + external_ids: + aemo_duid: [WDGPH1] + aemo_station_id: WDGPH + sources: [source:aemo:nem-registration-and-exemption-list] + notes: >- + Australia's largest operating solar farm (verified 460 MWp, ~1M panels, 22 km SE of Chinchilla); developed + and owned by Neoen (Neoen group acquired by Brookfield in 2025). Co-located Western Downs Battery + (270 MW, expanding) is a separate asset not modelled here. + + - entity_id: gen:au-nem:macintyre + entity_type: Generator + name: MacIntyre Wind Farm + aliases: [MacIntyre] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [151.40, -28.35] } + country: au + admin: { region: Queensland, locality: Karara } + grid_id: grid:au-nem + properties: + fuel: wind + capacity_mw: 923 + technology: onshore wind (162 x 5.7 MW Nordex Delta4000) + commissioned: 2024 + unit_count: 162 + status: operating + external_ids: {} + sources: [source:aemo:nem-registration-and-exemption-list] + notes: >- + Largest wind farm in the Southern Hemisphere (verified 923 MW; first power October 2024, fully operational + 2025). Acciona Energia majority owner with Ark Energy ~30%; the adjacent 103 MW CleanCo Karara wind farm is + part of the same precinct but separately owned. DUID omitted pending verification against the AEMO + registration list (staged registration across multiple units). diff --git a/intelligence/data/entities/au-nem/regions.yaml b/intelligence/data/entities/au-nem/regions.yaml new file mode 100644 index 0000000..82411e9 --- /dev/null +++ b/intelligence/data/entities/au-nem/regions.yaml @@ -0,0 +1,111 @@ +# Structural seed — NEM grid and dispatch regions. See ../../../schemas/entity.schema.json +records: + - entity_id: grid:au-nem + entity_type: Grid + name: National Electricity Market + aliases: [NEM] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [147.0, -32.0] } + country: au + grid_id: null + properties: + operator: AEMO + frequency_hz: 50 + regions: 5 + interconnected_customers_m: 23 + external_ids: {} + sources: [source:aemo:nem-registration-and-exemption-list, source:aemo:vis-nem-summary] + notes: >- + Interconnected east-coast Australian power system spanning QLD, NSW (incl. ACT), VIC, SA and TAS, + operated by the Australian Energy Market Operator. One of the world's longest AC systems (~5,000 km). + + - entity_id: region:au-nem:qld1 + entity_type: GridRegion + name: Queensland (QLD1) + aliases: [QLD1, Queensland region] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [146.0, -22.5] } + country: au + admin: { region: Queensland } + grid_id: grid:au-nem + properties: + peak_demand_mw: 11000 + demand_note: approximate record peak + external_ids: + aemo_region_id: QLD1 + sources: [source:aemo:nem-registration-and-exemption-list, source:aemo:vis-nem-summary] + notes: NEM dispatch region for Queensland. Hosts the Gladstone industrial cluster and the Curtis Island LNG export loads. + + - entity_id: region:au-nem:nsw1 + entity_type: GridRegion + name: New South Wales (NSW1) + aliases: [NSW1, New South Wales region] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [147.0, -32.5] } + country: au + admin: { region: New South Wales } + grid_id: grid:au-nem + properties: + peak_demand_mw: 14700 + demand_note: approximate record peak + external_ids: + aemo_region_id: NSW1 + sources: [source:aemo:nem-registration-and-exemption-list, source:aemo:vis-nem-summary] + notes: NEM dispatch region for New South Wales and the ACT. Largest regional demand; hosts Tomago smelter and the Sydney data-centre cluster. + + - entity_id: region:au-nem:vic1 + entity_type: GridRegion + name: Victoria (VIC1) + aliases: [VIC1, Victoria region] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [144.3, -36.9] } + country: au + admin: { region: Victoria } + grid_id: grid:au-nem + properties: + peak_demand_mw: 10500 + demand_note: approximate record peak + external_ids: + aemo_region_id: VIC1 + sources: [source:aemo:nem-registration-and-exemption-list, source:aemo:vis-nem-summary] + notes: NEM dispatch region for Victoria. + + - entity_id: region:au-nem:sa1 + entity_type: GridRegion + name: South Australia (SA1) + aliases: [SA1, South Australia region] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [135.8, -30.1] } + country: au + admin: { region: South Australia } + grid_id: grid:au-nem + properties: + peak_demand_mw: 3300 + demand_note: approximate record peak + external_ids: + aemo_region_id: SA1 + sources: [source:aemo:nem-registration-and-exemption-list, source:aemo:vis-nem-summary] + notes: NEM dispatch region for South Australia; world-leading variable-renewable penetration. + + - entity_id: region:au-nem:tas1 + entity_type: GridRegion + name: Tasmania (TAS1) + aliases: [TAS1, Tasmania region] + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [146.6, -42.0] } + country: au + admin: { region: Tasmania } + grid_id: grid:au-nem + properties: + peak_demand_mw: 1800 + demand_note: approximate record peak + external_ids: + aemo_region_id: TAS1 + sources: [source:aemo:nem-registration-and-exemption-list, source:aemo:vis-nem-summary] + notes: NEM dispatch region for Tasmania; predominantly hydro, connected to the mainland via Basslink. diff --git a/intelligence/data/entities/au-nem/vessels.yaml b/intelligence/data/entities/au-nem/vessels.yaml new file mode 100644 index 0000000..b15b77e --- /dev/null +++ b/intelligence/data/entities/au-nem/vessels.yaml @@ -0,0 +1,75 @@ +# Structural seed — SYNTHETIC exemplar vessels for demo scenarios. +# These are NOT real ships: IMO numbers 9000001-9000003 are deliberately outside the plausible +# registry range and properties.synthetic is true on every record. +# See ../../../schemas/entity.schema.json +records: + - entity_id: vessel:imo:9000001 + entity_type: Vessel + name: MV Bowen Trader + aliases: [] + data_class: structural + observability: ESTIMATED + geometry: null + country: null + grid_id: null + properties: + synthetic: true + vessel_type: capesize bulk carrier + dwt: 180000 + loa_m: 292 + beam_m: 45 + flag: pa + external_ids: + imo: "9000001" + sources: [source:gpc:cargo-statistics] + notes: >- + SYNTHETIC EXEMPLAR for demo purposes - not a real ship. Represents a typical capesize coal carrier + loading metallurgical coal at RG Tanna for Japan, modelled on trade patterns in GPC cargo statistics. + A real deployment would replace this with AIS-derived vessel entities (KNOWN_LIVE). + + - entity_id: vessel:imo:9000002 + entity_type: Vessel + name: MV Boyne Spirit + aliases: [] + data_class: structural + observability: ESTIMATED + geometry: null + country: null + grid_id: null + properties: + synthetic: true + vessel_type: general cargo / aluminium products carrier + dwt: 35000 + loa_m: 180 + beam_m: 30 + flag: sg + external_ids: + imo: "9000002" + sources: [source:gpc:cargo-statistics] + notes: >- + SYNTHETIC EXEMPLAR for demo purposes - not a real ship. Represents the regular aluminium ingot liner + trade from the Boyne Island smelter wharf to North Asian rolling mills. + + - entity_id: vessel:imo:9000003 + entity_type: Vessel + name: LNGC Curtis Pioneer + aliases: [] + data_class: structural + observability: ESTIMATED + geometry: null + country: null + grid_id: null + properties: + synthetic: true + vessel_type: LNG carrier (membrane) + dwt: 95000 + capacity_m3: 174000 + loa_m: 295 + beam_m: 46 + flag: bs + external_ids: + imo: "9000003" + sources: [source:gpc:cargo-statistics] + notes: >- + SYNTHETIC EXEMPLAR for demo purposes - not a real ship. Represents a standard 174,000 m3 LNG carrier + lifting a Curtis Island cargo for a Chinese term buyer. diff --git a/intelligence/data/relationships/au-nem/fuel-supply.yaml b/intelligence/data/relationships/au-nem/fuel-supply.yaml new file mode 100644 index 0000000..0e857ba --- /dev/null +++ b/intelligence/data/relationships/au-nem/fuel-supply.yaml @@ -0,0 +1,330 @@ +# Structural relationships — fuel supply: captive mines, coal contracts, CSG-to-LNG feed, +# basin membership and commodity production. See ../../../schemas/relationship.schema.json +records: + - rel_id: rel:callide-mine-supplies-callide-b + rel_type: SUPPLIES + from_id: mine:au-qld:callide + to_id: gen:au-nem:callide-b + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: "1988-01-01T00:00:00Z" + valid_to: null + properties: { commodity: commodity:coal-thermal, note: adjacent mine-mouth supply, long-term contract with CS Energy } + + - rel_id: rel:callide-mine-supplies-callide-c + rel_type: SUPPLIES + from_id: mine:au-qld:callide + to_id: gen:au-nem:callide-c + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: "2001-01-01T00:00:00Z" + valid_to: null + properties: { commodity: commodity:coal-thermal, note: adjacent mine-mouth supply } + + - rel_id: rel:meandu-supplies-tarong + rel_type: SUPPLIES + from_id: mine:au-qld:meandu + to_id: gen:au-nem:tarong + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: "1984-01-01T00:00:00Z" + valid_to: null + properties: { commodity: commodity:coal-thermal, note: captive mine owned by Stanwell Corporation } + + - rel_id: rel:meandu-supplies-tarong-north + rel_type: SUPPLIES + from_id: mine:au-qld:meandu + to_id: gen:au-nem:tarong-north + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: "2003-01-01T00:00:00Z" + valid_to: null + properties: { commodity: commodity:coal-thermal, note: captive mine owned by Stanwell Corporation } + + - rel_id: rel:kogan-creek-mine-supplies-kogan-creek-ps + rel_type: SUPPLIES + from_id: mine:au-qld:kogan-creek + to_id: gen:au-nem:kogan-creek + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: "2007-01-01T00:00:00Z" + valid_to: null + properties: { commodity: commodity:coal-thermal, note: CS Energy captive mine, conveyor-fed } + + - rel_id: rel:curragh-supplies-stanwell + rel_type: SUPPLIES + from_id: mine:au-qld:curragh + to_id: gen:au-nem:stanwell + confidence: 0.95 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: "1996-01-01T00:00:00Z" + valid_to: null + properties: + commodity: commodity:coal-thermal + note: >- + Public contract record (verified 2025): legacy amended coal supply agreement (~3 Mtpa) to early 2027, + replaced by a new agreement (~2 Mtpa) plus a June 2025 deal for up to 0.8 Mtpa over five years from + 2027 tied to Stanwell prepayment support for Coronado. + + - rel_id: rel:hvo-may-supply-bayswater + rel_type: MAY_SUPPLY + from_id: mine:au-nsw:hunter-valley-operations + to_id: gen:au-nem:bayswater + confidence: 0.5 + source: source:gem:global-coal-mine-tracker + method: derived + valid_from: null + valid_to: null + properties: + commodity: commodity:coal-thermal + note: >- + Hunter Valley domestic coal flows to Bayswater are commercially plausible and historically reported, + but current contract volumes are not public - modelled as MAY_SUPPLY, not a contract assertion. + + - rel_id: rel:surat-csg-supplies-qclng + rel_type: SUPPLIES + from_id: gasfield:au-qld:surat-csg + to_id: lng:au-qld:qclng + confidence: 1.0 + source: source:aemo:gas-bulletin-board + method: declared + valid_from: "2014-12-01T00:00:00Z" + valid_to: null + properties: { commodity: coal-seam gas, note: QGC-operated Surat fields; flows visible daily on the Gas Bulletin Board } + + - rel_id: rel:surat-csg-supplies-aplng + rel_type: SUPPLIES + from_id: gasfield:au-qld:surat-csg + to_id: lng:au-qld:aplng + confidence: 1.0 + source: source:aemo:gas-bulletin-board + method: declared + valid_from: "2015-12-01T00:00:00Z" + valid_to: null + properties: { commodity: coal-seam gas, note: APLNG-operated Surat fields (Talinga, Orana, Combabula and others) } + + - rel_id: rel:surat-csg-supplies-glng + rel_type: SUPPLIES + from_id: gasfield:au-qld:surat-csg + to_id: lng:au-qld:glng + confidence: 1.0 + source: source:aemo:gas-bulletin-board + method: declared + valid_from: "2015-09-01T00:00:00Z" + valid_to: null + properties: { commodity: coal-seam gas, note: GLNG equity gas (Fairview, Roma, Scotia) plus significant third-party purchases } + + - rel_id: rel:wgp-transports-to-qclng + rel_type: TRANSPORTS + from_id: pipe:au-qld:wallumbilla-gladstone + to_id: lng:au-qld:qclng + confidence: 1.0 + source: source:gem:global-gas-infrastructure-tracker + method: declared + valid_from: "2014-12-01T00:00:00Z" + valid_to: null + properties: { commodity: coal-seam gas } + + - rel_id: rel:aplng-pipeline-transports-to-aplng + rel_type: TRANSPORTS + from_id: pipe:au-qld:aplng-pipeline + to_id: lng:au-qld:aplng + confidence: 1.0 + source: source:gem:global-gas-infrastructure-tracker + method: declared + valid_from: "2015-12-01T00:00:00Z" + valid_to: null + properties: { commodity: coal-seam gas } + + - rel_id: rel:glng-gtp-transports-to-glng + rel_type: TRANSPORTS + from_id: pipe:au-qld:glng-gtp + to_id: lng:au-qld:glng + confidence: 1.0 + source: source:gem:global-gas-infrastructure-tracker + method: declared + valid_from: "2015-09-01T00:00:00Z" + valid_to: null + properties: { commodity: coal-seam gas } + + - rel_id: rel:blackwater-part-of-bowen + rel_type: PART_OF + from_id: mine:au-qld:blackwater + to_id: basin:au:bowen + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:goonyella-riverside-part-of-bowen + rel_type: PART_OF + from_id: mine:au-qld:goonyella-riverside + to_id: basin:au:bowen + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:peak-downs-part-of-bowen + rel_type: PART_OF + from_id: mine:au-qld:peak-downs + to_id: basin:au:bowen + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:curragh-part-of-bowen + rel_type: PART_OF + from_id: mine:au-qld:curragh + to_id: basin:au:bowen + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:kogan-creek-mine-part-of-surat + rel_type: PART_OF + from_id: mine:au-qld:kogan-creek + to_id: basin:au:surat + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: null + valid_to: null + properties: { note: Walloon coal measures } + + - rel_id: rel:surat-csg-part-of-surat + rel_type: PART_OF + from_id: gasfield:au-qld:surat-csg + to_id: basin:au:surat + confidence: 1.0 + source: source:gem:global-gas-infrastructure-tracker + method: declared + valid_from: null + valid_to: null + properties: { note: aggregate field entity; some GLNG fields (Fairview, Scotia) sit in the adjoining Bowen Basin } + + - rel_id: rel:blackwater-produces-met-coal + rel_type: PRODUCES + from_id: mine:au-qld:blackwater + to_id: commodity:coal-metallurgical + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:blackwater-produces-thermal-coal + rel_type: PRODUCES + from_id: mine:au-qld:blackwater + to_id: commodity:coal-thermal + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: null + valid_to: null + properties: { note: secondary product stream } + + - rel_id: rel:goonyella-riverside-produces-met-coal + rel_type: PRODUCES + from_id: mine:au-qld:goonyella-riverside + to_id: commodity:coal-metallurgical + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:peak-downs-produces-met-coal + rel_type: PRODUCES + from_id: mine:au-qld:peak-downs + to_id: commodity:coal-metallurgical + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:curragh-produces-met-coal + rel_type: PRODUCES + from_id: mine:au-qld:curragh + to_id: commodity:coal-metallurgical + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:curragh-produces-thermal-coal + rel_type: PRODUCES + from_id: mine:au-qld:curragh + to_id: commodity:coal-thermal + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: null + valid_to: null + properties: { note: domestic thermal stream to Stanwell } + + - rel_id: rel:callide-mine-produces-thermal-coal + rel_type: PRODUCES + from_id: mine:au-qld:callide + to_id: commodity:coal-thermal + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:meandu-produces-thermal-coal + rel_type: PRODUCES + from_id: mine:au-qld:meandu + to_id: commodity:coal-thermal + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:kogan-creek-mine-produces-thermal-coal + rel_type: PRODUCES + from_id: mine:au-qld:kogan-creek + to_id: commodity:coal-thermal + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:hvo-produces-thermal-coal + rel_type: PRODUCES + from_id: mine:au-nsw:hunter-valley-operations + to_id: commodity:coal-thermal + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: null + valid_to: null + properties: { note: also produces semi-soft coking coal } diff --git a/intelligence/data/relationships/au-nem/generation.yaml b/intelligence/data/relationships/au-nem/generation.yaml new file mode 100644 index 0000000..30653b6 --- /dev/null +++ b/intelligence/data/relationships/au-nem/generation.yaml @@ -0,0 +1,289 @@ +# Structural relationships — generator-to-region GENERATES edges (plus Wivenhoe pumping load). +# See ../../../schemas/relationship.schema.json +records: + - rel_id: rel:gladstone-ps-generates-qld1 + rel_type: GENERATES + from_id: gen:au-nem:gladstone-ps + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1976-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:callide-b-generates-qld1 + rel_type: GENERATES + from_id: gen:au-nem:callide-b + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1988-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:callide-c-generates-qld1 + rel_type: GENERATES + from_id: gen:au-nem:callide-c + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2001-01-01T00:00:00Z" + valid_to: null + properties: { note: C4 offline May 2021 - Aug 2024 after turbine hall explosion } + + - rel_id: rel:stanwell-generates-qld1 + rel_type: GENERATES + from_id: gen:au-nem:stanwell + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1993-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:kogan-creek-generates-qld1 + rel_type: GENERATES + from_id: gen:au-nem:kogan-creek + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2007-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:tarong-generates-qld1 + rel_type: GENERATES + from_id: gen:au-nem:tarong + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1984-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:tarong-north-generates-qld1 + rel_type: GENERATES + from_id: gen:au-nem:tarong-north + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2003-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:millmerran-generates-qld1 + rel_type: GENERATES + from_id: gen:au-nem:millmerran + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2002-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:wivenhoe-generates-qld1 + rel_type: GENERATES + from_id: gen:au-nem:wivenhoe + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1984-01-01T00:00:00Z" + valid_to: null + properties: { note: pumped storage - generates in peaks only } + + - rel_id: rel:wivenhoe-consumes-qld1 + rel_type: CONSUMES + from_id: gen:au-nem:wivenhoe + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: derived + valid_from: "1984-01-01T00:00:00Z" + valid_to: null + properties: + note: pumped-storage pumping load (~480 MW when pumping); net consumer of energy across a storage cycle + + - rel_id: rel:braemar-generates-qld1 + rel_type: GENERATES + from_id: gen:au-nem:braemar + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2006-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:darling-downs-generates-qld1 + rel_type: GENERATES + from_id: gen:au-nem:darling-downs + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2010-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:swanbank-e-generates-qld1 + rel_type: GENERATES + from_id: gen:au-nem:swanbank-e + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2002-01-01T00:00:00Z" + valid_to: null + properties: { note: mothballed 2017-2022; outage Apr-Oct 2025 } + + - rel_id: rel:mt-stuart-generates-qld1 + rel_type: GENERATES + from_id: gen:au-nem:mt-stuart + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1999-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:barron-gorge-generates-qld1 + rel_type: GENERATES + from_id: gen:au-nem:barron-gorge + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1963-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:kareeya-generates-qld1 + rel_type: GENERATES + from_id: gen:au-nem:kareeya + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1957-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:coopers-gap-generates-qld1 + rel_type: GENERATES + from_id: gen:au-nem:coopers-gap + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2020-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:western-downs-gph-generates-qld1 + rel_type: GENERATES + from_id: gen:au-nem:western-downs-gph + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2023-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:macintyre-generates-qld1 + rel_type: GENERATES + from_id: gen:au-nem:macintyre + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2024-10-01T00:00:00Z" + valid_to: null + properties: { note: first power October 2024, fully operational 2025 } + + - rel_id: rel:eraring-generates-nsw1 + rel_type: GENERATES + from_id: gen:au-nem:eraring + to_id: region:au-nem:nsw1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1982-01-01T00:00:00Z" + valid_to: null + properties: { note: closure extended to 30 April 2029 (announced Feb 2026) } + + - rel_id: rel:bayswater-generates-nsw1 + rel_type: GENERATES + from_id: gen:au-nem:bayswater + to_id: region:au-nem:nsw1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1985-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:mt-piper-generates-nsw1 + rel_type: GENERATES + from_id: gen:au-nem:mt-piper + to_id: region:au-nem:nsw1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1993-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:vales-point-generates-nsw1 + rel_type: GENERATES + from_id: gen:au-nem:vales-point + to_id: region:au-nem:nsw1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1978-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:tallawarra-generates-nsw1 + rel_type: GENERATES + from_id: gen:au-nem:tallawarra + to_id: region:au-nem:nsw1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2009-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:colongra-generates-nsw1 + rel_type: GENERATES + from_id: gen:au-nem:colongra + to_id: region:au-nem:nsw1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2009-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:uranquinty-generates-nsw1 + rel_type: GENERATES + from_id: gen:au-nem:uranquinty + to_id: region:au-nem:nsw1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2009-01-01T00:00:00Z" + valid_to: null + properties: {} diff --git a/intelligence/data/relationships/au-nem/gladstone-chain.yaml b/intelligence/data/relationships/au-nem/gladstone-chain.yaml new file mode 100644 index 0000000..1c40f24 --- /dev/null +++ b/intelligence/data/relationships/au-nem/gladstone-chain.yaml @@ -0,0 +1,436 @@ +# Structural relationships — the Gladstone demo backbone: power -> smelter, bauxite -> alumina -> +# aluminium -> export; coal rail -> terminals -> export; CSG -> LNG -> export; met coal -> steel demand. +# See ../../../schemas/relationship.schema.json +records: + - rel_id: rel:gladstone-ps-supplies-boyne-island + rel_type: SUPPLIES + from_id: gen:au-nem:gladstone-ps + to_id: smelter:au-qld:boyne-island + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "1982-01-01T00:00:00Z" + valid_to: null + properties: + commodity: electricity + note: >- + Public JV arrangement - most GPS output is committed to Boyne Smelters via dedicated 275 kV supply; + contracts run to March 2029 (verified 2025), which drives the GPS potential-retirement notice. + + - rel_id: rel:gladstone-ps-may-supply-qal + rel_type: MAY_SUPPLY + from_id: gen:au-nem:gladstone-ps + to_id: refinery:au-qld:qal + confidence: 0.5 + source: source:wikidata:sparql + method: derived + valid_from: null + valid_to: null + properties: { commodity: electricity, note: historically associated supply within the Rio Tinto Gladstone system; current commercial arrangements not public } + + - rel_id: rel:qal-produces-alumina + rel_type: PRODUCES + from_id: refinery:au-qld:qal + to_id: commodity:alumina + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "1967-01-01T00:00:00Z" + valid_to: null + properties: { capacity_tpa: 3950000 } + + - rel_id: rel:yarwun-produces-alumina + rel_type: PRODUCES + from_id: refinery:au-qld:yarwun + to_id: commodity:alumina + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2004-01-01T00:00:00Z" + valid_to: null + properties: { capacity_tpa: 3000000 } + + - rel_id: rel:qal-consumes-bauxite + rel_type: CONSUMES + from_id: refinery:au-qld:qal + to_id: commodity:bauxite + confidence: 1.0 + source: source:gpc:cargo-statistics + method: declared + valid_from: "1967-01-01T00:00:00Z" + valid_to: null + properties: { note: Weipa bauxite shipped down the Queensland coast; inbound tonnages visible in GPC statistics } + + - rel_id: rel:yarwun-consumes-bauxite + rel_type: CONSUMES + from_id: refinery:au-qld:yarwun + to_id: commodity:bauxite + confidence: 1.0 + source: source:gpc:cargo-statistics + method: declared + valid_from: "2004-01-01T00:00:00Z" + valid_to: null + properties: { note: Weipa bauxite feed } + + - rel_id: rel:qal-supplies-boyne-island + rel_type: SUPPLIES + from_id: refinery:au-qld:qal + to_id: smelter:au-qld:boyne-island + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "1982-01-01T00:00:00Z" + valid_to: null + properties: { commodity: commodity:alumina, note: alumina conveyed/shipped across Boyne estuary to the smelter; both plants sit in the Rio Tinto Gladstone system } + + - rel_id: rel:boyne-island-consumes-alumina + rel_type: CONSUMES + from_id: smelter:au-qld:boyne-island + to_id: commodity:alumina + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "1982-01-01T00:00:00Z" + valid_to: null + properties: { note: ~2 t alumina per tonne of metal } + + - rel_id: rel:boyne-island-produces-aluminium + rel_type: PRODUCES + from_id: smelter:au-qld:boyne-island + to_id: commodity:aluminium + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "1982-01-01T00:00:00Z" + valid_to: null + properties: { capacity_tpa: 500000, note: curtailed ~14% since 2017 } + + - rel_id: rel:boyne-island-exports-jp + rel_type: EXPORTS_TO + from_id: smelter:au-qld:boyne-island + to_id: dest:jp + confidence: 0.7 + source: source:gpc:cargo-statistics + method: derived + valid_from: null + valid_to: null + properties: { commodity: commodity:aluminium, note: public export-market reporting (JV partners include Japanese shareholders); not a contract assertion } + + - rel_id: rel:boyne-island-exports-kr + rel_type: EXPORTS_TO + from_id: smelter:au-qld:boyne-island + to_id: dest:kr + confidence: 0.7 + source: source:gpc:cargo-statistics + method: derived + valid_from: null + valid_to: null + properties: { commodity: commodity:aluminium, note: public export-market reporting; not a contract assertion } + + - rel_id: rel:blackwater-transports-rg-tanna + rel_type: TRANSPORTS + from_id: mine:au-qld:blackwater + to_id: terminal:au-qld:rg-tanna + confidence: 0.9 + source: source:gpc:cargo-statistics + method: derived + valid_from: null + valid_to: null + properties: { commodity: commodity:coal-metallurgical, via: rail:au-qld:blackwater-system } + + - rel_id: rel:curragh-transports-rg-tanna + rel_type: TRANSPORTS + from_id: mine:au-qld:curragh + to_id: terminal:au-qld:rg-tanna + confidence: 0.9 + source: source:gpc:cargo-statistics + method: derived + valid_from: null + valid_to: null + properties: { commodity: commodity:coal-metallurgical, via: rail:au-qld:blackwater-system } + + - rel_id: rel:curragh-transports-wicet + rel_type: TRANSPORTS + from_id: mine:au-qld:curragh + to_id: terminal:au-qld:wicet + confidence: 0.9 + source: source:gpc:cargo-statistics + method: derived + valid_from: "2015-01-01T00:00:00Z" + valid_to: null + properties: { commodity: commodity:coal-metallurgical, via: rail:au-qld:blackwater-system, note: Coronado Curragh is a WICET shareholder-shipper } + + - rel_id: rel:callide-mine-transports-rg-tanna + rel_type: TRANSPORTS + from_id: mine:au-qld:callide + to_id: terminal:au-qld:rg-tanna + confidence: 0.8 + source: source:gpc:cargo-statistics + method: derived + valid_from: null + valid_to: null + properties: { commodity: commodity:coal-thermal, via: rail:au-qld:moura-system, note: export surplus beyond mine-mouth power supply } + + - rel_id: rel:goonyella-riverside-transports-hpct + rel_type: TRANSPORTS + from_id: mine:au-qld:goonyella-riverside + to_id: terminal:au-qld:hay-point + confidence: 0.95 + source: source:wikidata:sparql + method: derived + valid_from: null + valid_to: null + properties: { commodity: commodity:coal-metallurgical, via: rail:au-qld:goonyella-system, note: BMA mine to BMA-owned terminal } + + - rel_id: rel:peak-downs-transports-hpct + rel_type: TRANSPORTS + from_id: mine:au-qld:peak-downs + to_id: terminal:au-qld:hay-point + confidence: 0.95 + source: source:wikidata:sparql + method: derived + valid_from: null + valid_to: null + properties: { commodity: commodity:coal-metallurgical, via: rail:au-qld:goonyella-system, note: BMA mine to BMA-owned terminal } + + - rel_id: rel:rg-tanna-part-of-gladstone-port + rel_type: PART_OF + from_id: terminal:au-qld:rg-tanna + to_id: port:au-qld:gladstone + confidence: 1.0 + source: source:gpc:cargo-statistics + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:wicet-part-of-gladstone-port + rel_type: PART_OF + from_id: terminal:au-qld:wicet + to_id: port:au-qld:gladstone + confidence: 1.0 + source: source:gpc:cargo-statistics + method: declared + valid_from: "2015-01-01T00:00:00Z" + valid_to: null + properties: { note: within port limits but independently owned } + + - rel_id: rel:barney-point-part-of-gladstone-port + rel_type: PART_OF + from_id: terminal:au-qld:barney-point + to_id: port:au-qld:gladstone + confidence: 1.0 + source: source:gpc:cargo-statistics + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:hpct-part-of-hay-point-port + rel_type: PART_OF + from_id: terminal:au-qld:hay-point + to_id: port:au-qld:hay-point + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:dbct-part-of-hay-point-port + rel_type: PART_OF + from_id: terminal:au-qld:dbct + to_id: port:au-qld:hay-point + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:qclng-loads-at-gladstone + rel_type: LOADS_AT + from_id: lng:au-qld:qclng + to_id: port:au-qld:gladstone + confidence: 1.0 + source: source:gpc:cargo-statistics + method: declared + valid_from: "2015-01-01T00:00:00Z" + valid_to: null + properties: { note: Curtis Island jetty within Port of Gladstone limits } + + - rel_id: rel:aplng-loads-at-gladstone + rel_type: LOADS_AT + from_id: lng:au-qld:aplng + to_id: port:au-qld:gladstone + confidence: 1.0 + source: source:gpc:cargo-statistics + method: declared + valid_from: "2016-01-01T00:00:00Z" + valid_to: null + properties: { note: Curtis Island jetty within Port of Gladstone limits } + + - rel_id: rel:glng-loads-at-gladstone + rel_type: LOADS_AT + from_id: lng:au-qld:glng + to_id: port:au-qld:gladstone + confidence: 1.0 + source: source:gpc:cargo-statistics + method: declared + valid_from: "2015-10-01T00:00:00Z" + valid_to: null + properties: { note: Curtis Island jetty within Port of Gladstone limits } + + - rel_id: rel:qclng-produces-lng + rel_type: PRODUCES + from_id: lng:au-qld:qclng + to_id: commodity:lng + confidence: 1.0 + source: source:gem:global-gas-infrastructure-tracker + method: declared + valid_from: "2014-12-01T00:00:00Z" + valid_to: null + properties: { capacity_mtpa: 8.5 } + + - rel_id: rel:aplng-produces-lng + rel_type: PRODUCES + from_id: lng:au-qld:aplng + to_id: commodity:lng + confidence: 1.0 + source: source:gem:global-gas-infrastructure-tracker + method: declared + valid_from: "2015-12-01T00:00:00Z" + valid_to: null + properties: { capacity_mtpa: 9.0 } + + - rel_id: rel:glng-produces-lng + rel_type: PRODUCES + from_id: lng:au-qld:glng + to_id: commodity:lng + confidence: 1.0 + source: source:gem:global-gas-infrastructure-tracker + method: declared + valid_from: "2015-09-01T00:00:00Z" + valid_to: null + properties: { capacity_mtpa: 7.8 } + + - rel_id: rel:qclng-exports-cn + rel_type: EXPORTS_TO + from_id: lng:au-qld:qclng + to_id: dest:cn + confidence: 0.8 + source: source:gpc:cargo-statistics + method: derived + valid_from: null + valid_to: null + properties: { commodity: commodity:lng, note: CNOOC anchor offtake; derived from public trade statistics } + + - rel_id: rel:qclng-exports-jp + rel_type: EXPORTS_TO + from_id: lng:au-qld:qclng + to_id: dest:jp + confidence: 0.7 + source: source:gpc:cargo-statistics + method: derived + valid_from: null + valid_to: null + properties: { commodity: commodity:lng, note: Tokyo Gas offtake plus portfolio cargoes } + + - rel_id: rel:aplng-exports-cn + rel_type: EXPORTS_TO + from_id: lng:au-qld:aplng + to_id: dest:cn + confidence: 0.8 + source: source:gpc:cargo-statistics + method: derived + valid_from: null + valid_to: null + properties: { commodity: commodity:lng, note: Sinopec anchor offtake } + + - rel_id: rel:aplng-exports-jp + rel_type: EXPORTS_TO + from_id: lng:au-qld:aplng + to_id: dest:jp + confidence: 0.7 + source: source:gpc:cargo-statistics + method: derived + valid_from: null + valid_to: null + properties: { commodity: commodity:lng, note: Kansai Electric and other Japanese buyers } + + - rel_id: rel:glng-exports-kr + rel_type: EXPORTS_TO + from_id: lng:au-qld:glng + to_id: dest:kr + confidence: 0.8 + source: source:gpc:cargo-statistics + method: derived + valid_from: null + valid_to: null + properties: { commodity: commodity:lng, note: KOGAS is a 15% partner and anchor offtaker (with PETRONAS to Malaysia, not modelled as a destination) } + + - rel_id: rel:orica-yarwun-may-supply-blackwater + rel_type: MAY_SUPPLY + from_id: chem:au-qld:orica-yarwun + to_id: mine:au-qld:blackwater + confidence: 0.6 + source: source:wikidata:sparql + method: derived + valid_from: null + valid_to: null + properties: { commodity: ammonium nitrate, note: Orica Yarwun is the principal AN source for Bowen Basin blasting; mine-level contracts not public } + + - rel_id: rel:orica-yarwun-may-supply-curragh + rel_type: MAY_SUPPLY + from_id: chem:au-qld:orica-yarwun + to_id: mine:au-qld:curragh + confidence: 0.6 + source: source:wikidata:sparql + method: derived + valid_from: null + valid_to: null + properties: { commodity: ammonium nitrate, note: mine-level contracts not public } + + - rel_id: rel:jp-steel-consumes-met-coal + rel_type: CONSUMES + from_id: market:jp:steel + to_id: commodity:coal-metallurgical + confidence: 0.9 + source: source:wikidata:sparql + method: derived + valid_from: null + valid_to: null + properties: { note: blast-furnace steelmaking; Australia supplies the majority of Japanese coking coal imports } + + - rel_id: rel:jp-steel-part-of-jp + rel_type: PART_OF + from_id: market:jp:steel + to_id: dest:jp + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:goonyella-riverside-may-supply-jp-steel + rel_type: MAY_SUPPLY + from_id: mine:au-qld:goonyella-riverside + to_id: market:jp:steel + confidence: 0.6 + source: source:wikidata:sparql + method: derived + valid_from: null + valid_to: null + properties: { commodity: commodity:coal-metallurgical, note: Mitsubishi JV alignment and long-run trade patterns; specific offtakes not public } + + - rel_id: rel:port-kembla-consumes-met-coal + rel_type: CONSUMES + from_id: steel:au-nsw:port-kembla + to_id: commodity:coal-metallurgical + confidence: 0.9 + source: source:wikidata:sparql + method: derived + valid_from: null + valid_to: null + properties: { note: primarily nearby Illawarra coking coal (mines not modelled in this seed) } diff --git a/intelligence/data/relationships/au-nem/grid-structure.yaml b/intelligence/data/relationships/au-nem/grid-structure.yaml new file mode 100644 index 0000000..16f20ed --- /dev/null +++ b/intelligence/data/relationships/au-nem/grid-structure.yaml @@ -0,0 +1,272 @@ +# Structural relationships — grid topology: regions, interconnectors, major loads. +# See ../../../schemas/relationship.schema.json +records: + - rel_id: rel:qld1-part-of-nem + rel_type: PART_OF + from_id: region:au-nem:qld1 + to_id: grid:au-nem + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1998-12-13T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:nsw1-part-of-nem + rel_type: PART_OF + from_id: region:au-nem:nsw1 + to_id: grid:au-nem + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1998-12-13T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:vic1-part-of-nem + rel_type: PART_OF + from_id: region:au-nem:vic1 + to_id: grid:au-nem + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1998-12-13T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:sa1-part-of-nem + rel_type: PART_OF + from_id: region:au-nem:sa1 + to_id: grid:au-nem + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1998-12-13T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:tas1-part-of-nem + rel_type: PART_OF + from_id: region:au-nem:tas1 + to_id: grid:au-nem + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2005-05-29T00:00:00Z" + valid_to: null + properties: { note: Tasmania joined the NEM in 2005, physically linked from 2006 via Basslink } + + - rel_id: rel:qni-interconnects-nsw1 + rel_type: INTERCONNECTED_WITH + from_id: intercon:au-nem:qni + to_id: region:au-nem:nsw1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2001-02-01T00:00:00Z" + valid_to: null + properties: { capacity_mw_into_region: 1078, capacity_mw_out_of_region: 600 } + + - rel_id: rel:qni-interconnects-qld1 + rel_type: INTERCONNECTED_WITH + from_id: intercon:au-nem:qni + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2001-02-01T00:00:00Z" + valid_to: null + properties: { capacity_mw_into_region: 600, capacity_mw_out_of_region: 1078 } + + - rel_id: rel:terranora-interconnects-nsw1 + rel_type: INTERCONNECTED_WITH + from_id: intercon:au-nem:terranora + to_id: region:au-nem:nsw1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2000-07-01T00:00:00Z" + valid_to: null + properties: { capacity_mw_into_region: 180, capacity_mw_out_of_region: 180 } + + - rel_id: rel:terranora-interconnects-qld1 + rel_type: INTERCONNECTED_WITH + from_id: intercon:au-nem:terranora + to_id: region:au-nem:qld1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2000-07-01T00:00:00Z" + valid_to: null + properties: { capacity_mw_into_region: 180, capacity_mw_out_of_region: 180 } + + - rel_id: rel:vni-interconnects-vic1 + rel_type: INTERCONNECTED_WITH + from_id: intercon:au-nem:vni + to_id: region:au-nem:vic1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1998-12-13T00:00:00Z" + valid_to: null + properties: { capacity_mw_into_region: 1350, capacity_mw_out_of_region: 1600 } + + - rel_id: rel:vni-interconnects-nsw1 + rel_type: INTERCONNECTED_WITH + from_id: intercon:au-nem:vni + to_id: region:au-nem:nsw1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1998-12-13T00:00:00Z" + valid_to: null + properties: { capacity_mw_into_region: 1600, capacity_mw_out_of_region: 1350 } + + - rel_id: rel:murraylink-interconnects-vic1 + rel_type: INTERCONNECTED_WITH + from_id: intercon:au-nem:murraylink + to_id: region:au-nem:vic1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2002-10-01T00:00:00Z" + valid_to: null + properties: { capacity_mw_into_region: 200, capacity_mw_out_of_region: 220 } + + - rel_id: rel:murraylink-interconnects-sa1 + rel_type: INTERCONNECTED_WITH + from_id: intercon:au-nem:murraylink + to_id: region:au-nem:sa1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2002-10-01T00:00:00Z" + valid_to: null + properties: { capacity_mw_into_region: 220, capacity_mw_out_of_region: 200 } + + - rel_id: rel:heywood-interconnects-vic1 + rel_type: INTERCONNECTED_WITH + from_id: intercon:au-nem:heywood + to_id: region:au-nem:vic1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1998-12-13T00:00:00Z" + valid_to: null + properties: { capacity_mw_into_region: 650, capacity_mw_out_of_region: 650 } + + - rel_id: rel:heywood-interconnects-sa1 + rel_type: INTERCONNECTED_WITH + from_id: intercon:au-nem:heywood + to_id: region:au-nem:sa1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "1998-12-13T00:00:00Z" + valid_to: null + properties: { capacity_mw_into_region: 650, capacity_mw_out_of_region: 650 } + + - rel_id: rel:basslink-interconnects-tas1 + rel_type: INTERCONNECTED_WITH + from_id: intercon:au-nem:basslink + to_id: region:au-nem:tas1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2006-04-29T00:00:00Z" + valid_to: null + properties: + capacity_mw_into_region: 478 + capacity_mw_out_of_region: 500 + note: APA-owned; converted from market network service to regulated (prescribed) transmission service, regulated operation from July 2026 (AER final decision June 2025) + + - rel_id: rel:basslink-interconnects-vic1 + rel_type: INTERCONNECTED_WITH + from_id: intercon:au-nem:basslink + to_id: region:au-nem:vic1 + confidence: 1.0 + source: source:aemo:nem-registration-and-exemption-list + method: declared + valid_from: "2006-04-29T00:00:00Z" + valid_to: null + properties: + capacity_mw_into_region: 500 + capacity_mw_out_of_region: 478 + note: up to ~630 MW short-term TAS->VIC + + - rel_id: rel:boyne-island-consumes-qld1 + rel_type: CONSUMES + from_id: smelter:au-qld:boyne-island + to_id: region:au-nem:qld1 + confidence: 0.95 + source: source:wikidata:sparql + method: derived + valid_from: "1982-01-01T00:00:00Z" + valid_to: null + properties: { load_mw: 950, note: largest single load in QLD1; supplied predominantly via Gladstone PS contracts to March 2029 } + + - rel_id: rel:tomago-consumes-nsw1 + rel_type: CONSUMES + from_id: smelter:au-nsw:tomago + to_id: region:au-nem:nsw1 + confidence: 0.95 + source: source:wikidata:sparql + method: derived + valid_from: "1983-01-01T00:00:00Z" + valid_to: null + properties: { load_mw: 950, note: about 10% of NSW demand; AGL is the current contracted supplier } + + - rel_id: rel:qal-consumes-qld1 + rel_type: CONSUMES + from_id: refinery:au-qld:qal + to_id: region:au-nem:qld1 + confidence: 0.9 + source: source:wikidata:sparql + method: derived + valid_from: "1967-01-01T00:00:00Z" + valid_to: null + properties: { note: large steam/electricity demand; partly self-supplied on site } + + - rel_id: rel:yarwun-consumes-qld1 + rel_type: CONSUMES + from_id: refinery:au-qld:yarwun + to_id: region:au-nem:qld1 + confidence: 0.9 + source: source:wikidata:sparql + method: derived + valid_from: "2004-01-01T00:00:00Z" + valid_to: null + properties: { note: has on-site gas cogeneration; net grid position varies } + + - rel_id: rel:orica-yarwun-consumes-qld1 + rel_type: CONSUMES + from_id: chem:au-qld:orica-yarwun + to_id: region:au-nem:qld1 + confidence: 0.85 + source: source:wikidata:sparql + method: derived + valid_from: "1990-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:port-kembla-consumes-nsw1 + rel_type: CONSUMES + from_id: steel:au-nsw:port-kembla + to_id: region:au-nem:nsw1 + confidence: 0.9 + source: source:wikidata:sparql + method: derived + valid_from: null + valid_to: null + properties: { note: integrated steelworks; significant on-site generation from process gases } + + - rel_id: rel:sydney-dc-cluster-consumes-nsw1 + rel_type: CONSUMES + from_id: dc:au-nsw:sydney-metro-cluster + to_id: region:au-nem:nsw1 + confidence: 0.9 + source: source:wikidata:sparql + method: derived + valid_from: null + valid_to: null + properties: { load_mw: 1100, note: aggregate estimate; fastest-growing large load in NSW1 } diff --git a/intelligence/data/relationships/au-nem/logistics.yaml b/intelligence/data/relationships/au-nem/logistics.yaml new file mode 100644 index 0000000..84b55b6 --- /dev/null +++ b/intelligence/data/relationships/au-nem/logistics.yaml @@ -0,0 +1,267 @@ +# Structural relationships — export logistics: port destination flows, rail-port connectivity, +# and SYNTHETIC exemplar vessel movements (demo scenarios, not real ship tracks). +# See ../../../schemas/relationship.schema.json +records: + - rel_id: rel:vessel-bowen-trader-loads-at-rg-tanna + rel_type: LOADS_AT + from_id: vessel:imo:9000001 + to_id: terminal:au-qld:rg-tanna + confidence: 1.0 + source: source:gpc:cargo-statistics + method: manual + valid_from: null + valid_to: null + properties: { commodity: commodity:coal-metallurgical, note: SYNTHETIC demo scenario - exemplar vessel, not a real ship movement } + + - rel_id: rel:vessel-bowen-trader-departs-gladstone + rel_type: DEPARTS_FROM + from_id: vessel:imo:9000001 + to_id: port:au-qld:gladstone + confidence: 1.0 + source: source:gpc:cargo-statistics + method: manual + valid_from: null + valid_to: null + properties: { note: SYNTHETIC demo scenario } + + - rel_id: rel:vessel-bowen-trader-ships-to-jp + rel_type: SHIPS_TO + from_id: vessel:imo:9000001 + to_id: dest:jp + confidence: 1.0 + source: source:gpc:cargo-statistics + method: manual + valid_from: null + valid_to: null + properties: { commodity: commodity:coal-metallurgical, note: SYNTHETIC demo scenario } + + - rel_id: rel:vessel-boyne-spirit-loads-at-gladstone + rel_type: LOADS_AT + from_id: vessel:imo:9000002 + to_id: port:au-qld:gladstone + confidence: 1.0 + source: source:gpc:cargo-statistics + method: manual + valid_from: null + valid_to: null + properties: { commodity: commodity:aluminium, note: SYNTHETIC demo scenario - loads aluminium ingot at the Boyne wharf } + + - rel_id: rel:vessel-boyne-spirit-ships-to-jp + rel_type: SHIPS_TO + from_id: vessel:imo:9000002 + to_id: dest:jp + confidence: 1.0 + source: source:gpc:cargo-statistics + method: manual + valid_from: null + valid_to: null + properties: { commodity: commodity:aluminium, note: SYNTHETIC demo scenario } + + - rel_id: rel:vessel-curtis-pioneer-loads-at-gladstone + rel_type: LOADS_AT + from_id: vessel:imo:9000003 + to_id: port:au-qld:gladstone + confidence: 1.0 + source: source:gpc:cargo-statistics + method: manual + valid_from: null + valid_to: null + properties: { commodity: commodity:lng, note: SYNTHETIC demo scenario - Curtis Island LNG jetty } + + - rel_id: rel:vessel-curtis-pioneer-ships-to-cn + rel_type: SHIPS_TO + from_id: vessel:imo:9000003 + to_id: dest:cn + confidence: 1.0 + source: source:gpc:cargo-statistics + method: manual + valid_from: null + valid_to: null + properties: { commodity: commodity:lng, note: SYNTHETIC demo scenario } + + - rel_id: rel:gladstone-port-exports-jp + rel_type: EXPORTS_TO + from_id: port:au-qld:gladstone + to_id: dest:jp + confidence: 0.9 + source: source:gpc:cargo-statistics + method: derived + valid_from: null + valid_to: null + properties: { commodities: [commodity:coal-metallurgical, commodity:coal-thermal, commodity:lng, commodity:aluminium, commodity:alumina], note: derived from GPC destination statistics } + + - rel_id: rel:gladstone-port-exports-cn + rel_type: EXPORTS_TO + from_id: port:au-qld:gladstone + to_id: dest:cn + confidence: 0.9 + source: source:gpc:cargo-statistics + method: derived + valid_from: null + valid_to: null + properties: { commodities: [commodity:lng, commodity:coal-metallurgical, commodity:alumina], note: derived from GPC destination statistics } + + - rel_id: rel:gladstone-port-exports-kr + rel_type: EXPORTS_TO + from_id: port:au-qld:gladstone + to_id: dest:kr + confidence: 0.9 + source: source:gpc:cargo-statistics + method: derived + valid_from: null + valid_to: null + properties: { commodities: [commodity:coal-thermal, commodity:coal-metallurgical, commodity:lng, commodity:aluminium], note: derived from GPC destination statistics } + + - rel_id: rel:gladstone-port-exports-tw + rel_type: EXPORTS_TO + from_id: port:au-qld:gladstone + to_id: dest:tw + confidence: 0.85 + source: source:gpc:cargo-statistics + method: derived + valid_from: null + valid_to: null + properties: { commodities: [commodity:coal-thermal, commodity:coal-metallurgical], note: derived from GPC destination statistics } + + - rel_id: rel:gladstone-port-exports-in + rel_type: EXPORTS_TO + from_id: port:au-qld:gladstone + to_id: dest:in + confidence: 0.85 + source: source:gpc:cargo-statistics + method: derived + valid_from: null + valid_to: null + properties: { commodities: [commodity:coal-metallurgical], note: derived from GPC destination statistics } + + - rel_id: rel:hay-point-exports-jp + rel_type: EXPORTS_TO + from_id: port:au-qld:hay-point + to_id: dest:jp + confidence: 0.85 + source: source:wikidata:sparql + method: derived + valid_from: null + valid_to: null + properties: { commodities: [commodity:coal-metallurgical], note: derived from public trade statistics } + + - rel_id: rel:hay-point-exports-cn + rel_type: EXPORTS_TO + from_id: port:au-qld:hay-point + to_id: dest:cn + confidence: 0.8 + source: source:wikidata:sparql + method: derived + valid_from: null + valid_to: null + properties: { commodities: [commodity:coal-metallurgical], note: flows resumed after the 2020-23 informal import ban } + + - rel_id: rel:hay-point-exports-in + rel_type: EXPORTS_TO + from_id: port:au-qld:hay-point + to_id: dest:in + confidence: 0.85 + source: source:wikidata:sparql + method: derived + valid_from: null + valid_to: null + properties: { commodities: [commodity:coal-metallurgical] } + + - rel_id: rel:hay-point-exports-kr + rel_type: EXPORTS_TO + from_id: port:au-qld:hay-point + to_id: dest:kr + confidence: 0.85 + source: source:wikidata:sparql + method: derived + valid_from: null + valid_to: null + properties: { commodities: [commodity:coal-metallurgical] } + + - rel_id: rel:newcastle-exports-jp + rel_type: EXPORTS_TO + from_id: port:au-nsw:newcastle + to_id: dest:jp + confidence: 0.9 + source: source:wikidata:sparql + method: derived + valid_from: null + valid_to: null + properties: { commodities: [commodity:coal-thermal], note: Japan is the largest Newcastle thermal coal destination } + + - rel_id: rel:newcastle-exports-cn + rel_type: EXPORTS_TO + from_id: port:au-nsw:newcastle + to_id: dest:cn + confidence: 0.8 + source: source:wikidata:sparql + method: derived + valid_from: null + valid_to: null + properties: { commodities: [commodity:coal-thermal] } + + - rel_id: rel:newcastle-exports-tw + rel_type: EXPORTS_TO + from_id: port:au-nsw:newcastle + to_id: dest:tw + confidence: 0.85 + source: source:wikidata:sparql + method: derived + valid_from: null + valid_to: null + properties: { commodities: [commodity:coal-thermal] } + + - rel_id: rel:newcastle-exports-kr + rel_type: EXPORTS_TO + from_id: port:au-nsw:newcastle + to_id: dest:kr + confidence: 0.85 + source: source:wikidata:sparql + method: derived + valid_from: null + valid_to: null + properties: { commodities: [commodity:coal-thermal] } + + - rel_id: rel:hvo-transports-newcastle + rel_type: TRANSPORTS + from_id: mine:au-nsw:hunter-valley-operations + to_id: port:au-nsw:newcastle + confidence: 0.9 + source: source:gem:global-coal-mine-tracker + method: derived + valid_from: null + valid_to: null + properties: { commodity: commodity:coal-thermal, note: via the Hunter Valley rail corridor (ARTC network, not modelled) to PWCS/NCIG terminals } + + - rel_id: rel:blackwater-system-connected-gladstone + rel_type: CONNECTED_TO + from_id: rail:au-qld:blackwater-system + to_id: port:au-qld:gladstone + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: null + valid_to: null + properties: { note: terminates at RG Tanna and WICET } + + - rel_id: rel:moura-system-connected-gladstone + rel_type: CONNECTED_TO + from_id: rail:au-qld:moura-system + to_id: port:au-qld:gladstone + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:goonyella-system-connected-hay-point + rel_type: CONNECTED_TO + from_id: rail:au-qld:goonyella-system + to_id: port:au-qld:hay-point + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: null + valid_to: null + properties: { note: terminates at HPCT and DBCT } diff --git a/intelligence/data/relationships/au-nem/ownership.yaml b/intelligence/data/relationships/au-nem/ownership.yaml new file mode 100644 index 0000000..74a46fa --- /dev/null +++ b/intelligence/data/relationships/au-nem/ownership.yaml @@ -0,0 +1,723 @@ +# Structural relationships — ownership and operatorship. JV shares recorded in properties; +# JV nuance in notes. See ../../../schemas/relationship.schema.json +records: + - rel_id: rel:boyne-island-owned-by-rio-tinto + rel_type: OWNED_BY + from_id: smelter:au-qld:boyne-island + to_id: company:au:rio-tinto + confidence: 0.95 + source: source:wikidata:sparql + method: declared + valid_from: null + valid_to: null + properties: { share: "73.5%", note: majority JV; minority partners incl. YKK, UACJ and Southern Cross (Japanese interests); Rio Tinto operates } + + - rel_id: rel:gladstone-ps-operated-by-nrg + rel_type: OPERATED_BY + from_id: gen:au-nem:gladstone-ps + to_id: company:au:nrg-gladstone + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "1994-03-01T00:00:00Z" + valid_to: null + properties: { note: NRG Gladstone Operating Services operates for the unincorporated JV since privatisation in 1994 } + + - rel_id: rel:gladstone-ps-owned-by-rio-tinto + rel_type: OWNED_BY + from_id: gen:au-nem:gladstone-ps + to_id: company:au:rio-tinto + confidence: 0.9 + source: source:wikidata:sparql + method: declared + valid_from: "1994-03-01T00:00:00Z" + valid_to: null + properties: + share: "42.125%" + note: >- + Unincorporated JV (verified 2025): Rio Tinto 42.125%, NRG Energy 37.5%, Southern Cross GPS 8.25%, + Ryowa II GPS 7.125%, YKK GPS 4.75%. CS Energy acts as market intermediary for output. + + - rel_id: rel:gladstone-ps-owned-by-nrg + rel_type: OWNED_BY + from_id: gen:au-nem:gladstone-ps + to_id: company:au:nrg-gladstone + confidence: 0.85 + source: source:wikidata:sparql + method: declared + valid_from: "1994-03-01T00:00:00Z" + valid_to: null + properties: { share: "37.5%", note: NRG Energy Inc JV interest, represented here by its Australian entity } + + - rel_id: rel:qal-owned-by-rio-tinto + rel_type: OWNED_BY + from_id: refinery:au-qld:qal + to_id: company:au:rio-tinto + confidence: 0.95 + source: source:wikidata:sparql + method: declared + valid_from: null + valid_to: null + properties: { share: "80%", note: Rusal 20% quarantined under Australian sanctions; Rio Tinto has had 100% operational control since 2022 } + + - rel_id: rel:yarwun-owned-by-rio-tinto + rel_type: OWNED_BY + from_id: refinery:au-qld:yarwun + to_id: company:au:rio-tinto + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2004-01-01T00:00:00Z" + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:orica-yarwun-owned-by-orica + rel_type: OWNED_BY + from_id: chem:au-qld:orica-yarwun + to_id: company:au:orica + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:callide-b-owned-by-cs-energy + rel_type: OWNED_BY + from_id: gen:au-nem:callide-b + to_id: company:au:cs-energy + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: null + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:callide-c-owned-by-cs-energy + rel_type: OWNED_BY + from_id: gen:au-nem:callide-c + to_id: company:au:cs-energy + confidence: 0.95 + source: source:wikidata:sparql + method: declared + valid_from: "2001-01-01T00:00:00Z" + valid_to: null + properties: { share: "50%" } + + - rel_id: rel:callide-c-owned-by-sev-en-gi + rel_type: OWNED_BY + from_id: gen:au-nem:callide-c + to_id: company:au:sev-en-gi + confidence: 0.9 + source: source:wikidata:sparql + method: declared + valid_from: "2025-05-01T00:00:00Z" + valid_to: null + properties: + share: "50%" + note: >- + Verified May 2025 - Sev.en GI acquired and recapitalised the 50% formerly held by IG Power/Genuity + (in administration from March 2023). Sev.en GI has publicly disputed pre-acquisition losses with CS Energy. + + - rel_id: rel:callide-c-operated-by-cs-energy + rel_type: OPERATED_BY + from_id: gen:au-nem:callide-c + to_id: company:au:cs-energy + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2001-01-01T00:00:00Z" + valid_to: null + properties: {} + + - rel_id: rel:kogan-creek-ps-owned-by-cs-energy + rel_type: OWNED_BY + from_id: gen:au-nem:kogan-creek + to_id: company:au:cs-energy + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2007-01-01T00:00:00Z" + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:kogan-creek-mine-owned-by-cs-energy + rel_type: OWNED_BY + from_id: mine:au-qld:kogan-creek + to_id: company:au:cs-energy + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2007-01-01T00:00:00Z" + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:stanwell-owned-by-stanwell-corp + rel_type: OWNED_BY + from_id: gen:au-nem:stanwell + to_id: company:au:stanwell-corp + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: null + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:tarong-owned-by-stanwell-corp + rel_type: OWNED_BY + from_id: gen:au-nem:tarong + to_id: company:au:stanwell-corp + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2011-07-01T00:00:00Z" + valid_to: null + properties: { share: "100%", note: transferred from Tarong Energy in the 2011 Queensland genco restructure } + + - rel_id: rel:tarong-north-owned-by-stanwell-corp + rel_type: OWNED_BY + from_id: gen:au-nem:tarong-north + to_id: company:au:stanwell-corp + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2011-07-01T00:00:00Z" + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:meandu-owned-by-stanwell-corp + rel_type: OWNED_BY + from_id: mine:au-qld:meandu + to_id: company:au:stanwell-corp + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: null + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:wivenhoe-owned-by-cleanco + rel_type: OWNED_BY + from_id: gen:au-nem:wivenhoe + to_id: company:au:cleanco-qld + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2019-10-31T00:00:00Z" + valid_to: null + properties: { share: "100%", note: transferred from CS Energy at CleanCo commencement } + + - rel_id: rel:swanbank-e-owned-by-cleanco + rel_type: OWNED_BY + from_id: gen:au-nem:swanbank-e + to_id: company:au:cleanco-qld + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2019-10-31T00:00:00Z" + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:barron-gorge-owned-by-cleanco + rel_type: OWNED_BY + from_id: gen:au-nem:barron-gorge + to_id: company:au:cleanco-qld + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2019-10-31T00:00:00Z" + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:kareeya-owned-by-cleanco + rel_type: OWNED_BY + from_id: gen:au-nem:kareeya + to_id: company:au:cleanco-qld + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2019-10-31T00:00:00Z" + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:millmerran-operated-by-intergen + rel_type: OPERATED_BY + from_id: gen:au-nem:millmerran + to_id: company:au:intergen-au + confidence: 0.9 + source: source:wikidata:sparql + method: declared + valid_from: "2002-01-01T00:00:00Z" + valid_to: null + properties: { note: InterGen operates for the Millmerran Power Partners consortium (ownership shares not modelled - multi-party financial consortium) } + + - rel_id: rel:braemar-owned-by-alinta + rel_type: OWNED_BY + from_id: gen:au-nem:braemar + to_id: company:au:alinta-energy + confidence: 0.95 + source: source:wikidata:sparql + method: declared + valid_from: "2008-01-01T00:00:00Z" + valid_to: null + properties: { share: "100%", note: Braemar 1 only (verified ex-ERM Power); the adjacent Braemar 2 is Arrow Energy's and is out of scope } + + - rel_id: rel:darling-downs-owned-by-origin + rel_type: OWNED_BY + from_id: gen:au-nem:darling-downs + to_id: company:au:origin-energy + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2010-01-01T00:00:00Z" + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:mt-stuart-owned-by-origin + rel_type: OWNED_BY + from_id: gen:au-nem:mt-stuart + to_id: company:au:origin-energy + confidence: 0.95 + source: source:wikidata:sparql + method: declared + valid_from: null + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:coopers-gap-operated-by-agl + rel_type: OPERATED_BY + from_id: gen:au-nem:coopers-gap + to_id: company:au:agl-energy + confidence: 0.95 + source: source:wikidata:sparql + method: declared + valid_from: "2020-01-01T00:00:00Z" + valid_to: null + properties: { note: owned by the Powering Australian Renewables partnership (AGL 20% with QIC/Future Fund); AGL operates and offtakes } + + - rel_id: rel:western-downs-gph-owned-by-neoen + rel_type: OWNED_BY + from_id: gen:au-nem:western-downs-gph + to_id: company:au:neoen + confidence: 0.9 + source: source:wikidata:sparql + method: declared + valid_from: "2023-01-01T00:00:00Z" + valid_to: null + properties: { share: "100%", note: Neoen group acquired by Brookfield in 2025; asset-level ownership assumed unchanged } + + - rel_id: rel:macintyre-owned-by-acciona + rel_type: OWNED_BY + from_id: gen:au-nem:macintyre + to_id: company:au:acciona-energia + confidence: 0.9 + source: source:wikidata:sparql + method: declared + valid_from: "2024-10-01T00:00:00Z" + valid_to: null + properties: { share: "~70%", note: co-owned with Ark Energy (~30%, verified); Acciona Energia operates } + + - rel_id: rel:eraring-owned-by-origin + rel_type: OWNED_BY + from_id: gen:au-nem:eraring + to_id: company:au:origin-energy + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2013-01-01T00:00:00Z" + valid_to: null + properties: { share: "100%", note: NSW Government underwriting agreement covers extended operation to April 2029 } + + - rel_id: rel:bayswater-owned-by-agl + rel_type: OWNED_BY + from_id: gen:au-nem:bayswater + to_id: company:au:agl-energy + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2014-01-01T00:00:00Z" + valid_to: null + properties: { share: "100%", note: via AGL Macquarie acquisition from NSW Government } + + - rel_id: rel:mt-piper-owned-by-energyaustralia + rel_type: OWNED_BY + from_id: gen:au-nem:mt-piper + to_id: company:au:energyaustralia + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2013-01-01T00:00:00Z" + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:vales-point-owned-by-delta + rel_type: OWNED_BY + from_id: gen:au-nem:vales-point + to_id: company:au:delta-electricity + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2015-11-01T00:00:00Z" + valid_to: null + properties: { share: "100%", note: privatised by NSW in 2015 for $1m; Delta acquired by Sev.en GI in 2022 } + + - rel_id: rel:delta-owned-by-sev-en-gi + rel_type: OWNED_BY + from_id: company:au:delta-electricity + to_id: company:au:sev-en-gi + confidence: 0.95 + source: source:wikidata:sparql + method: declared + valid_from: "2022-08-01T00:00:00Z" + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:tallawarra-owned-by-energyaustralia + rel_type: OWNED_BY + from_id: gen:au-nem:tallawarra + to_id: company:au:energyaustralia + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2009-01-01T00:00:00Z" + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:colongra-owned-by-snowy-hydro + rel_type: OWNED_BY + from_id: gen:au-nem:colongra + to_id: company:au:snowy-hydro + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2014-01-01T00:00:00Z" + valid_to: null + properties: { share: "100%", note: acquired from Delta Electricity in 2014 } + + - rel_id: rel:uranquinty-owned-by-origin + rel_type: OWNED_BY + from_id: gen:au-nem:uranquinty + to_id: company:au:origin-energy + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2008-01-01T00:00:00Z" + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:rg-tanna-owned-by-gpc + rel_type: OWNED_BY + from_id: terminal:au-qld:rg-tanna + to_id: company:au:gladstone-ports-corp + confidence: 1.0 + source: source:gpc:cargo-statistics + method: declared + valid_from: null + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:barney-point-owned-by-gpc + rel_type: OWNED_BY + from_id: terminal:au-qld:barney-point + to_id: company:au:gladstone-ports-corp + confidence: 1.0 + source: source:gpc:cargo-statistics + method: declared + valid_from: null + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:gladstone-port-operated-by-gpc + rel_type: OPERATED_BY + from_id: port:au-qld:gladstone + to_id: company:au:gladstone-ports-corp + confidence: 1.0 + source: source:gpc:cargo-statistics + method: declared + valid_from: null + valid_to: null + properties: { note: Queensland government-owned port authority } + + - rel_id: rel:wicet-owned-by-wicet-holdings + rel_type: OWNED_BY + from_id: terminal:au-qld:wicet + to_id: company:au:wicet-holdings + confidence: 0.95 + source: source:wikidata:sparql + method: declared + valid_from: "2015-01-01T00:00:00Z" + valid_to: null + properties: { note: shipper consortium - Glencore ~40% with Coronado Curragh and Yancoal (verified); independent of GPC } + + - rel_id: rel:hpct-owned-by-bma + rel_type: OWNED_BY + from_id: terminal:au-qld:hay-point + to_id: company:au:bhp-mitsubishi-alliance + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: null + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:dbct-operated-by-dbi + rel_type: OPERATED_BY + from_id: terminal:au-qld:dbct + to_id: company:au:dalrymple-bay-infrastructure + confidence: 0.95 + source: source:wikidata:sparql + method: declared + valid_from: null + valid_to: null + properties: { note: terminal is Queensland government-owned; DBI holds a 99-year operating lease } + + - rel_id: rel:blackwater-system-operated-by-aurizon + rel_type: OPERATED_BY + from_id: rail:au-qld:blackwater-system + to_id: company:au:aurizon + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: null + valid_to: null + properties: { note: Aurizon Network owns and operates the CQCN under regulated access } + + - rel_id: rel:moura-system-operated-by-aurizon + rel_type: OPERATED_BY + from_id: rail:au-qld:moura-system + to_id: company:au:aurizon + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:goonyella-system-operated-by-aurizon + rel_type: OPERATED_BY + from_id: rail:au-qld:goonyella-system + to_id: company:au:aurizon + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: null + valid_to: null + properties: {} + + - rel_id: rel:qclng-owned-by-shell-qgc + rel_type: OWNED_BY + from_id: lng:au-qld:qclng + to_id: company:au:shell-qgc + confidence: 0.95 + source: source:gem:global-gas-infrastructure-tracker + method: declared + valid_from: "2016-02-01T00:00:00Z" + valid_to: null + properties: { note: Shell (via QGC) majority owner-operator; CNOOC 50% of Train 1, Tokyo Gas 2.5% of Train 2 } + + - rel_id: rel:aplng-owned-by-origin + rel_type: OWNED_BY + from_id: lng:au-qld:aplng + to_id: company:au:origin-energy + confidence: 0.95 + source: source:gem:global-gas-infrastructure-tracker + method: declared + valid_from: "2015-12-01T00:00:00Z" + valid_to: null + properties: { share: "27.5%", note: reduced from 37.5% in 2022; Origin operates the upstream fields } + + - rel_id: rel:aplng-owned-by-conocophillips + rel_type: OWNED_BY + from_id: lng:au-qld:aplng + to_id: company:au:conocophillips + confidence: 0.95 + source: source:gem:global-gas-infrastructure-tracker + method: declared + valid_from: "2015-12-01T00:00:00Z" + valid_to: null + properties: { share: "47.5%", note: Sinopec holds the remaining 25% (no separate company entity in this seed) } + + - rel_id: rel:aplng-operated-by-conocophillips + rel_type: OPERATED_BY + from_id: lng:au-qld:aplng + to_id: company:au:conocophillips + confidence: 0.95 + source: source:gem:global-gas-infrastructure-tracker + method: declared + valid_from: "2015-12-01T00:00:00Z" + valid_to: null + properties: { note: ConocoPhillips operates the downstream LNG facility; Origin operates upstream } + + - rel_id: rel:glng-owned-by-santos + rel_type: OWNED_BY + from_id: lng:au-qld:glng + to_id: company:au:santos + confidence: 0.95 + source: source:gem:global-gas-infrastructure-tracker + method: declared + valid_from: "2015-09-01T00:00:00Z" + valid_to: null + properties: { share: "30%", note: Santos is operator; PETRONAS 27.5%, TotalEnergies 27.5%, KOGAS 15% } + + - rel_id: rel:wgp-owned-by-apa + rel_type: OWNED_BY + from_id: pipe:au-qld:wallumbilla-gladstone + to_id: company:au:apa-group + confidence: 1.0 + source: source:gem:global-gas-infrastructure-tracker + method: declared + valid_from: "2015-06-01T00:00:00Z" + valid_to: null + properties: { share: "100%", note: acquired from BG Group in 2015 } + + - rel_id: rel:aplng-pipeline-owned-by-origin + rel_type: OWNED_BY + from_id: pipe:au-qld:aplng-pipeline + to_id: company:au:origin-energy + confidence: 0.6 + source: source:gem:global-gas-infrastructure-tracker + method: derived + valid_from: null + valid_to: null + properties: { note: pipeline is owned by the APLNG JV; edge assigned to upstream operator Origin as JV proxy - see APLNG ownership edges for the full JV } + + - rel_id: rel:glng-gtp-owned-by-santos + rel_type: OWNED_BY + from_id: pipe:au-qld:glng-gtp + to_id: company:au:santos + confidence: 0.8 + source: source:gem:global-gas-infrastructure-tracker + method: derived + valid_from: null + valid_to: null + properties: { note: owned by the GLNG JV; edge assigned to operator Santos as JV proxy } + + - rel_id: rel:goonyella-riverside-owned-by-bma + rel_type: OWNED_BY + from_id: mine:au-qld:goonyella-riverside + to_id: company:au:bhp-mitsubishi-alliance + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: null + valid_to: null + properties: { share: "100%", note: BMA itself is a 50/50 BHP / Mitsubishi Development JV } + + - rel_id: rel:peak-downs-owned-by-bma + rel_type: OWNED_BY + from_id: mine:au-qld:peak-downs + to_id: company:au:bhp-mitsubishi-alliance + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: null + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:blackwater-owned-by-whitehaven + rel_type: OWNED_BY + from_id: mine:au-qld:blackwater + to_id: company:au:whitehaven-coal + confidence: 0.95 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: "2024-04-02T00:00:00Z" + valid_to: null + properties: + share: "70%" + note: >- + Verified - acquired from BMA April 2024; 30% sold to Nippon Steel (20%) and JFE Steel (10%) effective + 1 April 2025. Whitehaven manages the unincorporated JV. + + - rel_id: rel:curragh-owned-by-coronado + rel_type: OWNED_BY + from_id: mine:au-qld:curragh + to_id: company:au:coronado + confidence: 0.95 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: "2018-01-01T00:00:00Z" + valid_to: null + properties: { share: "100%", note: acquired from Wesfarmers 2018; Coronado under liquidity pressure through 2025 - ownership could change, monitor } + + - rel_id: rel:callide-mine-owned-by-batchfire + rel_type: OWNED_BY + from_id: mine:au-qld:callide + to_id: company:au:batchfire-resources + confidence: 0.95 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: "2016-10-31T00:00:00Z" + valid_to: null + properties: { share: "100%" } + + - rel_id: rel:callide-mine-owned-by-anglo-historical + rel_type: OWNED_BY + from_id: mine:au-qld:callide + to_id: company:au:anglo-american + confidence: 1.0 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: "2000-01-01T00:00:00Z" + valid_to: "2016-10-31T00:00:00Z" + properties: { note: historical edge - Anglo American sold Callide to Batchfire Resources in October 2016 } + + - rel_id: rel:hvo-owned-by-yancoal + rel_type: OWNED_BY + from_id: mine:au-nsw:hunter-valley-operations + to_id: company:au:yancoal + confidence: 0.95 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: "2017-09-01T00:00:00Z" + valid_to: null + properties: { share: "51%", note: Yancoal operates the HVO JV } + + - rel_id: rel:hvo-owned-by-glencore + rel_type: OWNED_BY + from_id: mine:au-nsw:hunter-valley-operations + to_id: company:au:glencore + confidence: 0.95 + source: source:gem:global-coal-mine-tracker + method: declared + valid_from: "2017-09-01T00:00:00Z" + valid_to: null + properties: { share: "49%" } + + - rel_id: rel:tomago-owned-by-tac + rel_type: OWNED_BY + from_id: smelter:au-nsw:tomago + to_id: company:au:tomago-aluminium + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "1983-01-01T00:00:00Z" + valid_to: null + properties: { note: TAC JV - Rio Tinto ~51.55%, CSR ~36.05%, Hydro Aluminium ~12.4% } + + - rel_id: rel:port-kembla-owned-by-bluescope + rel_type: OWNED_BY + from_id: steel:au-nsw:port-kembla + to_id: company:au:bluescope + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2002-07-01T00:00:00Z" + valid_to: null + properties: { share: "100%", note: since the BHP Steel spin-off } + + - rel_id: rel:basslink-owned-by-apa + rel_type: OWNED_BY + from_id: intercon:au-nem:basslink + to_id: company:au:apa-group + confidence: 1.0 + source: source:wikidata:sparql + method: declared + valid_from: "2022-10-01T00:00:00Z" + valid_to: null + properties: + share: "100%" + note: >- + Verified - APA acquired Basslink Pty Ltd out of administration in 2022 (it is APA-owned, not + state-owned); AER approved conversion to a prescribed regulated transmission service in June 2025, + effective as a regulated asset from July 2026. From 3517faddf0fb21a70a25c4d991ae4ca4ca78b3c7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 01:59:25 +0000 Subject: [PATCH 12/16] feat(intelligence): connector SDK + AEMO/AIS/BOM/ENTSO-E/EIA connectors (phases 6, 8, 12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Connector contract (discover/fetch/parse/normalize/verify) behind an HttpGate seam where fixture replay is the same code path as live fetching. AEMO NEMWEB DispatchIS/Dispatch_SCADA/TradingIS (zip + MMS C/I/D CSV, NEM-time conversion, RUNNO-keyed corrections), the AEMO dashboard summary API, aisstream websocket (Gladstone bbox, unmapped MMSIs dropped and reported — never guessed), BOM observations, and ENTSO-E/EIA breadth stubs. All fixtures are synthetic-from-spec and say so; verify-live harness writes a report and never edits registries. Deterministic fixture generator; 32 tests. Seed vessels gain MMSIs matching the AIS fixtures. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014yVGqBtvM1zNAd9tvUVNzm --- .../data/entities/au-nem/vessels.yaml | 3 + .../aemo-nemweb-dispatch-scada/README.md | 14 + .../exchanges/00-dir-listing.json | 6 + .../exchanges/01-dispatchscada-zip.json | 6 + .../fixtures/aemo-nemweb-dispatchis/README.md | 14 + .../exchanges/00-dir-listing.json | 6 + .../exchanges/01-dispatchis-zip.json | 6 + .../fixtures/aemo-nemweb-tradingis/README.md | 14 + .../exchanges/00-dir-listing.json | 6 + .../exchanges/01-tradingis-zip.json | 6 + .../fixtures/aemo-vis-nem-summary/README.md | 14 + .../exchanges/00-elec-nem-summary.json | 6 + .../connectors/fixtures/aisstream/README.md | 14 + .../fixtures/aisstream/messages.jsonl | 20 + .../fixtures/bom-observations/README.md | 14 + .../exchanges/00-idq60801-gladstone.json | 6 + .../connectors/fixtures/eia-v2/README.md | 14 + .../eia-v2/exchanges/00-region-data.json | 6 + .../fixtures/entsoe-transparency/README.md | 14 + .../exchanges/00-a75-sample.json | 6 + intelligence/packages/connectors/package.json | 7 +- .../connectors/scripts/make-fixtures.mjs | 517 ++++++++++++++++++ .../packages/connectors/src/aemo-csv.ts | 110 ++++ intelligence/packages/connectors/src/cli.ts | 178 ++++++ .../connectors/aemo-nemweb-dispatch-scada.ts | 76 +++ .../src/connectors/aemo-nemweb-dispatchis.ts | 149 +++++ .../src/connectors/aemo-nemweb-tradingis.ts | 79 +++ .../src/connectors/aemo-vis-nem-summary.ts | 149 +++++ .../connectors/src/connectors/aisstream.ts | 250 +++++++++ .../src/connectors/bom-observations.ts | 181 ++++++ .../connectors/src/connectors/eia-v2.ts | 148 +++++ .../src/connectors/entsoe-transparency.ts | 184 +++++++ .../connectors/src/connectors/index.ts | 37 ++ .../src/connectors/nemweb-common.ts | 76 +++ .../packages/connectors/src/entity-index.ts | 74 +++ intelligence/packages/connectors/src/gate.ts | 172 ++++++ intelligence/packages/connectors/src/index.ts | 15 +- .../packages/connectors/src/observations.ts | 58 ++ .../packages/connectors/src/scheduler.ts | 130 +++++ intelligence/packages/connectors/src/sdk.ts | 95 ++++ .../packages/connectors/src/verify-live.ts | 104 ++++ intelligence/packages/connectors/src/zip.ts | 22 + .../connectors/test/aemo-csv.test.mjs | 78 +++ .../connectors/test/connectors-aemo.test.mjs | 113 ++++ .../test/connectors-stream.test.mjs | 96 ++++ .../connectors/test/connectors-stubs.test.mjs | 75 +++ .../packages/connectors/test/e2e.test.mjs | 82 +++ .../packages/connectors/test/gate.test.mjs | 51 ++ .../packages/connectors/test/helpers.mjs | 53 ++ .../packages/connectors/test/zip.test.mjs | 19 + 50 files changed, 3570 insertions(+), 3 deletions(-) create mode 100644 intelligence/packages/connectors/fixtures/aemo-nemweb-dispatch-scada/README.md create mode 100644 intelligence/packages/connectors/fixtures/aemo-nemweb-dispatch-scada/exchanges/00-dir-listing.json create mode 100644 intelligence/packages/connectors/fixtures/aemo-nemweb-dispatch-scada/exchanges/01-dispatchscada-zip.json create mode 100644 intelligence/packages/connectors/fixtures/aemo-nemweb-dispatchis/README.md create mode 100644 intelligence/packages/connectors/fixtures/aemo-nemweb-dispatchis/exchanges/00-dir-listing.json create mode 100644 intelligence/packages/connectors/fixtures/aemo-nemweb-dispatchis/exchanges/01-dispatchis-zip.json create mode 100644 intelligence/packages/connectors/fixtures/aemo-nemweb-tradingis/README.md create mode 100644 intelligence/packages/connectors/fixtures/aemo-nemweb-tradingis/exchanges/00-dir-listing.json create mode 100644 intelligence/packages/connectors/fixtures/aemo-nemweb-tradingis/exchanges/01-tradingis-zip.json create mode 100644 intelligence/packages/connectors/fixtures/aemo-vis-nem-summary/README.md create mode 100644 intelligence/packages/connectors/fixtures/aemo-vis-nem-summary/exchanges/00-elec-nem-summary.json create mode 100644 intelligence/packages/connectors/fixtures/aisstream/README.md create mode 100644 intelligence/packages/connectors/fixtures/aisstream/messages.jsonl create mode 100644 intelligence/packages/connectors/fixtures/bom-observations/README.md create mode 100644 intelligence/packages/connectors/fixtures/bom-observations/exchanges/00-idq60801-gladstone.json create mode 100644 intelligence/packages/connectors/fixtures/eia-v2/README.md create mode 100644 intelligence/packages/connectors/fixtures/eia-v2/exchanges/00-region-data.json create mode 100644 intelligence/packages/connectors/fixtures/entsoe-transparency/README.md create mode 100644 intelligence/packages/connectors/fixtures/entsoe-transparency/exchanges/00-a75-sample.json create mode 100644 intelligence/packages/connectors/scripts/make-fixtures.mjs create mode 100644 intelligence/packages/connectors/src/aemo-csv.ts create mode 100644 intelligence/packages/connectors/src/cli.ts create mode 100644 intelligence/packages/connectors/src/connectors/aemo-nemweb-dispatch-scada.ts create mode 100644 intelligence/packages/connectors/src/connectors/aemo-nemweb-dispatchis.ts create mode 100644 intelligence/packages/connectors/src/connectors/aemo-nemweb-tradingis.ts create mode 100644 intelligence/packages/connectors/src/connectors/aemo-vis-nem-summary.ts create mode 100644 intelligence/packages/connectors/src/connectors/aisstream.ts create mode 100644 intelligence/packages/connectors/src/connectors/bom-observations.ts create mode 100644 intelligence/packages/connectors/src/connectors/eia-v2.ts create mode 100644 intelligence/packages/connectors/src/connectors/entsoe-transparency.ts create mode 100644 intelligence/packages/connectors/src/connectors/index.ts create mode 100644 intelligence/packages/connectors/src/connectors/nemweb-common.ts create mode 100644 intelligence/packages/connectors/src/entity-index.ts create mode 100644 intelligence/packages/connectors/src/gate.ts create mode 100644 intelligence/packages/connectors/src/observations.ts create mode 100644 intelligence/packages/connectors/src/scheduler.ts create mode 100644 intelligence/packages/connectors/src/sdk.ts create mode 100644 intelligence/packages/connectors/src/verify-live.ts create mode 100644 intelligence/packages/connectors/src/zip.ts create mode 100644 intelligence/packages/connectors/test/aemo-csv.test.mjs create mode 100644 intelligence/packages/connectors/test/connectors-aemo.test.mjs create mode 100644 intelligence/packages/connectors/test/connectors-stream.test.mjs create mode 100644 intelligence/packages/connectors/test/connectors-stubs.test.mjs create mode 100644 intelligence/packages/connectors/test/e2e.test.mjs create mode 100644 intelligence/packages/connectors/test/gate.test.mjs create mode 100644 intelligence/packages/connectors/test/helpers.mjs create mode 100644 intelligence/packages/connectors/test/zip.test.mjs diff --git a/intelligence/data/entities/au-nem/vessels.yaml b/intelligence/data/entities/au-nem/vessels.yaml index b15b77e..135a413 100644 --- a/intelligence/data/entities/au-nem/vessels.yaml +++ b/intelligence/data/entities/au-nem/vessels.yaml @@ -21,6 +21,7 @@ records: flag: pa external_ids: imo: "9000001" + mmsi: "503000001" sources: [source:gpc:cargo-statistics] notes: >- SYNTHETIC EXEMPLAR for demo purposes - not a real ship. Represents a typical capesize coal carrier @@ -45,6 +46,7 @@ records: flag: sg external_ids: imo: "9000002" + mmsi: "503000002" sources: [source:gpc:cargo-statistics] notes: >- SYNTHETIC EXEMPLAR for demo purposes - not a real ship. Represents the regular aluminium ingot liner @@ -69,6 +71,7 @@ records: flag: bs external_ids: imo: "9000003" + mmsi: "503000003" sources: [source:gpc:cargo-statistics] notes: >- SYNTHETIC EXEMPLAR for demo purposes - not a real ship. Represents a standard 174,000 m3 LNG carrier diff --git a/intelligence/packages/connectors/fixtures/aemo-nemweb-dispatch-scada/README.md b/intelligence/packages/connectors/fixtures/aemo-nemweb-dispatch-scada/README.md new file mode 100644 index 0000000..654b138 --- /dev/null +++ b/intelligence/packages/connectors/fixtures/aemo-nemweb-dispatch-scada/README.md @@ -0,0 +1,14 @@ +# Fixtures — AEMO NEMWEB Dispatch_SCADA + +fixture_provenance: synthetic-from-spec + +Every file in this directory is SYNTHETIC test data, generated by +`scripts/make-fixtures.mjs` from the source's published format +specification. No fixture here is real recorded data, and nothing in this +package claims live verification of the source (egress-blocked build; see +docs/LEGAL-LICENSING.md and the workspace honesty rules). + +Format spec bound against: AEMO MMS Data Model report CSV, table DISPATCH.UNIT_SCADA (SETTLEMENTDATE, DUID, SCADAVALUE) in a single-entry ZIP. GSTONE1-6 and CALL_B_1 map to seed generators; NOMAP_1 exercises unmapped reporting. + +Regenerate with: `npm run make-fixtures -w @pact-tailor/connectors` +(deterministic — fixed timestamps around 2026-08-12T01:00Z, no randomness). diff --git a/intelligence/packages/connectors/fixtures/aemo-nemweb-dispatch-scada/exchanges/00-dir-listing.json b/intelligence/packages/connectors/fixtures/aemo-nemweb-dispatch-scada/exchanges/00-dir-listing.json new file mode 100644 index 0000000..762a111 --- /dev/null +++ b/intelligence/packages/connectors/fixtures/aemo-nemweb-dispatch-scada/exchanges/00-dir-listing.json @@ -0,0 +1,6 @@ +{ + "url_pattern": "https://nemweb.com.au/Reports/Current/Dispatch_SCADA/", + "status": 200, + "content_type": "text/html", + "body_base64": "PGh0bWw+PGhlYWQ+PHRpdGxlPm5lbXdlYi5jb20uYXUgLSAvUmVwb3J0cy9DdXJyZW50L0Rpc3BhdGNoX1NDQURBLzwvdGl0bGU+PC9oZWFkPjxib2R5PjxIMT5uZW13ZWIuY29tLmF1IC0gL1JlcG9ydHMvQ3VycmVudC9EaXNwYXRjaF9TQ0FEQS88L0gxPjxocj48cHJlPjxBIEhSRUY9Ii9SZXBvcnRzL0N1cnJlbnQvIj5bVG8gUGFyZW50IERpcmVjdG9yeV08L0E+PGJyPjxBIEhSRUY9Ii9SZXBvcnRzL0N1cnJlbnQvRGlzcGF0Y2hfU0NBREEvUFVCTElDX0RJU1BBVENIU0NBREFfMjAyNjA4MTIxMTA1XzAwMDAwMDAxMjM0NTY3OTEuemlwIj5QVUJMSUNfRElTUEFUQ0hTQ0FEQV8yMDI2MDgxMjExMDVfMDAwMDAwMDEyMzQ1Njc5MS56aXA8L0E+PGJyPjwvcHJlPjxocj48L2JvZHk+PC9odG1sPg==" +} diff --git a/intelligence/packages/connectors/fixtures/aemo-nemweb-dispatch-scada/exchanges/01-dispatchscada-zip.json b/intelligence/packages/connectors/fixtures/aemo-nemweb-dispatch-scada/exchanges/01-dispatchscada-zip.json new file mode 100644 index 0000000..f0a0f2e --- /dev/null +++ b/intelligence/packages/connectors/fixtures/aemo-nemweb-dispatch-scada/exchanges/01-dispatchscada-zip.json @@ -0,0 +1,6 @@ +{ + "url_pattern": "https://nemweb.com.au/Reports/Current/Dispatch_SCADA/PUBLIC_DISPATCHSCADA_202608121105_0000000123456791.zip", + "status": 200, + "content_type": "application/x-zip-compressed", + "body_base64": "UEsDBBQAAAAIAKAIDF3NYobc1AAAAKgCAAA2AAAAUFVCTElDX0RJU1BBVENIU0NBREFfMjAyNjA4MTIxMTA1XzAwMDAwMDAxMjM0NTY3OTEuQ1NWnZBBbsIwEEX3SNwhynpkPJM4pOyMPW0tObaVOO0y979FDVQg2OHZzn/6T99A4CmJ3zh7C9YtSWfzvRhtNWieIqT17J0BkjQc5HhAAsSTVCfZgbwdUter4fiBL/jLW+537p6ANbi83XIIC+fseeKQrc4MdnUWrr8f7Vfe7x5iz1z7sGr+rWQLX0uOgRGoQ6FKaw1NQDSKi3MNXaYRspLti7cSWEkroJ7EUdXRQ+kexfB2t9Heb+cNoUMS/dt4iJNOhVbqupqBloNt4mczc4pzbuGyxh9QSwECFAAUAAAACACgCAxdzWKG3NQAAACoAgAANgAAAAAAAAAAAAAAAAAAAAAAUFVCTElDX0RJU1BBVENIU0NBREFfMjAyNjA4MTIxMTA1XzAwMDAwMDAxMjM0NTY3OTEuQ1NWUEsFBgAAAAABAAEAZAAAACgBAAAAAA==" +} diff --git a/intelligence/packages/connectors/fixtures/aemo-nemweb-dispatchis/README.md b/intelligence/packages/connectors/fixtures/aemo-nemweb-dispatchis/README.md new file mode 100644 index 0000000..a3247d9 --- /dev/null +++ b/intelligence/packages/connectors/fixtures/aemo-nemweb-dispatchis/README.md @@ -0,0 +1,14 @@ +# Fixtures — AEMO NEMWEB DispatchIS + +fixture_provenance: synthetic-from-spec + +Every file in this directory is SYNTHETIC test data, generated by +`scripts/make-fixtures.mjs` from the source's published format +specification. No fixture here is real recorded data, and nothing in this +package claims live verification of the source (egress-blocked build; see +docs/LEGAL-LICENSING.md and the workspace honesty rules). + +Format spec bound against: AEMO MMS Data Model report CSV (C/I/D rows), tables DISPATCH.PRICE / DISPATCH.REGIONSUM / DISPATCH.INTERCONNECTORRES, in a single-entry ZIP behind an IIS-style directory listing. The NSW1 RUNNO=2/INTERVENTION=1 PRICE row exercises correction marking; SA1 and MADEUP-X1 exercise unmapped reporting (against the unit tests' inline entity index). + +Regenerate with: `npm run make-fixtures -w @pact-tailor/connectors` +(deterministic — fixed timestamps around 2026-08-12T01:00Z, no randomness). diff --git a/intelligence/packages/connectors/fixtures/aemo-nemweb-dispatchis/exchanges/00-dir-listing.json b/intelligence/packages/connectors/fixtures/aemo-nemweb-dispatchis/exchanges/00-dir-listing.json new file mode 100644 index 0000000..9623628 --- /dev/null +++ b/intelligence/packages/connectors/fixtures/aemo-nemweb-dispatchis/exchanges/00-dir-listing.json @@ -0,0 +1,6 @@ +{ + "url_pattern": "https://nemweb.com.au/Reports/Current/DispatchIS_Reports/", + "status": 200, + "content_type": "text/html", + "body_base64": "PGh0bWw+PGhlYWQ+PHRpdGxlPm5lbXdlYi5jb20uYXUgLSAvUmVwb3J0cy9DdXJyZW50L0Rpc3BhdGNoSVNfUmVwb3J0cy88L3RpdGxlPjwvaGVhZD48Ym9keT48SDE+bmVtd2ViLmNvbS5hdSAtIC9SZXBvcnRzL0N1cnJlbnQvRGlzcGF0Y2hJU19SZXBvcnRzLzwvSDE+PGhyPjxwcmU+PEEgSFJFRj0iL1JlcG9ydHMvQ3VycmVudC8iPltUbyBQYXJlbnQgRGlyZWN0b3J5XTwvQT48YnI+PEEgSFJFRj0iL1JlcG9ydHMvQ3VycmVudC9EaXNwYXRjaElTX1JlcG9ydHMvUFVCTElDX0RJU1BBVENISVNfMjAyNjA4MTIxMTAwXzAwMDAwMDAxMjM0NTY3MDAuemlwIj5QVUJMSUNfRElTUEFUQ0hJU18yMDI2MDgxMjExMDBfMDAwMDAwMDEyMzQ1NjcwMC56aXA8L0E+PGJyPjxBIEhSRUY9Ii9SZXBvcnRzL0N1cnJlbnQvRGlzcGF0Y2hJU19SZXBvcnRzL1BVQkxJQ19ESVNQQVRDSElTXzIwMjYwODEyMTEwNV8wMDAwMDAwMTIzNDU2Nzg5LnppcCI+UFVCTElDX0RJU1BBVENISVNfMjAyNjA4MTIxMTA1XzAwMDAwMDAxMjM0NTY3ODkuemlwPC9BPjxicj48L3ByZT48aHI+PC9ib2R5PjwvaHRtbD4=" +} diff --git a/intelligence/packages/connectors/fixtures/aemo-nemweb-dispatchis/exchanges/01-dispatchis-zip.json b/intelligence/packages/connectors/fixtures/aemo-nemweb-dispatchis/exchanges/01-dispatchis-zip.json new file mode 100644 index 0000000..428bf3c --- /dev/null +++ b/intelligence/packages/connectors/fixtures/aemo-nemweb-dispatchis/exchanges/01-dispatchis-zip.json @@ -0,0 +1,6 @@ +{ + "url_pattern": "https://nemweb.com.au/Reports/Current/DispatchIS_Reports/PUBLIC_DISPATCHIS_202608121105_0000000123456789.zip", + "status": 200, + "content_type": "application/x-zip-compressed", + "body_base64": "UEsDBBQAAAAIAKAIDF2g94j2hAEAANQEAAAzAAAAUFVCTElDX0RJU1BBVENISVNfMjAyNjA4MTIxMTA1XzAwMDAwMDAxMjM0NTY3ODkuQ1NWpVLLksIgELxb5T9YnscIBDTxhmF0U0WABdT9/y9ZDL6i1m5lN4dMyDDdM93TgMHOFSfrtQLVBidj89EGkNhZcIetbhtghK2WpFpSBpRuiNgQDiQ/lJVcrNZV/Vj7lKumk/aWBufbBoFDwBg1dmiikhHBH4yx4HHfWtM+dGIi+qPUkD/S7ZQH7x0gOvDWgXTNTsv9dHIvunHM753PLp2TOVD41Ir2U5GKMlqWQKDihSD3OArOhNMzXM0Keoa7xBFw7BWOQi2KtTjD5TgQNGsWDt0/RY02Sq2wk0aBPMpWy63GPRr0ss9fq59+G4w9TEqYPQ4GfexshBWrtDgFF1AxQgpCQNQpJhUXpTifR1O8sSctbIJkUNNMsRY9NPyRIchnAsozILsQUH6JLBM8+JfVs8ZgE633GKB87+Pw4q9+dpgOqLrTTtsT3IK2IWAYjPiug5/EXLwxreRVwa4CpjELPpRxDEcnFR7c4utF1Izdv88bMZ00MEejZnY38+isj6m6nE6+AVBLAQIUABQAAAAIAKAIDF2g94j2hAEAANQEAAAzAAAAAAAAAAAAAAAAAAAAAABQVUJMSUNfRElTUEFUQ0hJU18yMDI2MDgxMjExMDVfMDAwMDAwMDEyMzQ1Njc4OS5DU1ZQSwUGAAAAAAEAAQBhAAAA1QEAAAAA" +} diff --git a/intelligence/packages/connectors/fixtures/aemo-nemweb-tradingis/README.md b/intelligence/packages/connectors/fixtures/aemo-nemweb-tradingis/README.md new file mode 100644 index 0000000..2afd292 --- /dev/null +++ b/intelligence/packages/connectors/fixtures/aemo-nemweb-tradingis/README.md @@ -0,0 +1,14 @@ +# Fixtures — AEMO NEMWEB TradingIS + +fixture_provenance: synthetic-from-spec + +Every file in this directory is SYNTHETIC test data, generated by +`scripts/make-fixtures.mjs` from the source's published format +specification. No fixture here is real recorded data, and nothing in this +package claims live verification of the source (egress-blocked build; see +docs/LEGAL-LICENSING.md and the workspace honesty rules). + +Format spec bound against: AEMO MMS Data Model report CSV, table TRADING.PRICE (trading-interval RRP per region) in a single-entry ZIP. SA1 exercises unmapped reporting against the unit tests' inline index. + +Regenerate with: `npm run make-fixtures -w @pact-tailor/connectors` +(deterministic — fixed timestamps around 2026-08-12T01:00Z, no randomness). diff --git a/intelligence/packages/connectors/fixtures/aemo-nemweb-tradingis/exchanges/00-dir-listing.json b/intelligence/packages/connectors/fixtures/aemo-nemweb-tradingis/exchanges/00-dir-listing.json new file mode 100644 index 0000000..23bb1fa --- /dev/null +++ b/intelligence/packages/connectors/fixtures/aemo-nemweb-tradingis/exchanges/00-dir-listing.json @@ -0,0 +1,6 @@ +{ + "url_pattern": "https://nemweb.com.au/Reports/Current/TradingIS_Reports/", + "status": 200, + "content_type": "text/html", + "body_base64": "PGh0bWw+PGhlYWQ+PHRpdGxlPm5lbXdlYi5jb20uYXUgLSAvUmVwb3J0cy9DdXJyZW50L1RyYWRpbmdJU19SZXBvcnRzLzwvdGl0bGU+PC9oZWFkPjxib2R5PjxIMT5uZW13ZWIuY29tLmF1IC0gL1JlcG9ydHMvQ3VycmVudC9UcmFkaW5nSVNfUmVwb3J0cy88L0gxPjxocj48cHJlPjxBIEhSRUY9Ii9SZXBvcnRzL0N1cnJlbnQvIj5bVG8gUGFyZW50IERpcmVjdG9yeV08L0E+PGJyPjxBIEhSRUY9Ii9SZXBvcnRzL0N1cnJlbnQvVHJhZGluZ0lTX1JlcG9ydHMvUFVCTElDX1RSQURJTkdJU18yMDI2MDgxMjExMDVfMDAwMDAwMDEyMzQ1Njc5Mi56aXAiPlBVQkxJQ19UUkFESU5HSVNfMjAyNjA4MTIxMTA1XzAwMDAwMDAxMjM0NTY3OTIuemlwPC9BPjxicj48L3ByZT48aHI+PC9ib2R5PjwvaHRtbD4=" +} diff --git a/intelligence/packages/connectors/fixtures/aemo-nemweb-tradingis/exchanges/01-tradingis-zip.json b/intelligence/packages/connectors/fixtures/aemo-nemweb-tradingis/exchanges/01-tradingis-zip.json new file mode 100644 index 0000000..92753e3 --- /dev/null +++ b/intelligence/packages/connectors/fixtures/aemo-nemweb-tradingis/exchanges/01-tradingis-zip.json @@ -0,0 +1,6 @@ +{ + "url_pattern": "https://nemweb.com.au/Reports/Current/TradingIS_Reports/PUBLIC_TRADINGIS_202608121105_0000000123456792.zip", + "status": 200, + "content_type": "application/x-zip-compressed", + "body_base64": "UEsDBBQAAAAIAKAIDF1UWEolvQAAAH0BAAAyAAAAUFVCTElDX1RSQURJTkdJU18yMDI2MDgxMjExMDVfMDAwMDAwMDEyMzQ1Njc5Mi5DU1aVjkFugzAQRfdI3AGx/iK2SSB05+IJGsmM3cFp7n+TojRNq+4yM4uRvt7XmyG05u6WNAYU9YFl4Q2e1oR8fY88wxk3HMz5YB2sfTOn/WC+x7r+eBrGyf1B/0W2rvgnRVaeCT02KiXSSlKCLwS9iiQoLZyEAzIpp7A/qhlEGSyfPnK4RL/U1VPz2db+GjYPQ9PC4iMGCzviPHTOYN9XYNlud3jqu+PL8Obv7Gi76cHOaElCky6NUk5aWox19QVQSwECFAAUAAAACACgCAxdVFhKJb0AAAB9AQAAMgAAAAAAAAAAAAAAAAAAAAAAUFVCTElDX1RSQURJTkdJU18yMDI2MDgxMjExMDVfMDAwMDAwMDEyMzQ1Njc5Mi5DU1ZQSwUGAAAAAAEAAQBgAAAADQEAAAAA" +} diff --git a/intelligence/packages/connectors/fixtures/aemo-vis-nem-summary/README.md b/intelligence/packages/connectors/fixtures/aemo-vis-nem-summary/README.md new file mode 100644 index 0000000..1dd405d --- /dev/null +++ b/intelligence/packages/connectors/fixtures/aemo-vis-nem-summary/README.md @@ -0,0 +1,14 @@ +# Fixtures — AEMO visualisations ELEC_NEM_SUMMARY + +fixture_provenance: synthetic-from-spec + +Every file in this directory is SYNTHETIC test data, generated by +`scripts/make-fixtures.mjs` from the source's published format +specification. No fixture here is real recorded data, and nothing in this +package claims live verification of the source (egress-blocked build; see +docs/LEGAL-LICENSING.md and the workspace honesty rules). + +Format spec bound against: JSON shape of AEMO's public NEM data dashboard backing endpoint (documented-from-public-dashboard; no formal API spec — see the connector's header comment and the source registry record). SA1 exercises unmapped reporting against the unit tests' inline index. + +Regenerate with: `npm run make-fixtures -w @pact-tailor/connectors` +(deterministic — fixed timestamps around 2026-08-12T01:00Z, no randomness). diff --git a/intelligence/packages/connectors/fixtures/aemo-vis-nem-summary/exchanges/00-elec-nem-summary.json b/intelligence/packages/connectors/fixtures/aemo-vis-nem-summary/exchanges/00-elec-nem-summary.json new file mode 100644 index 0000000..d9081d2 --- /dev/null +++ b/intelligence/packages/connectors/fixtures/aemo-vis-nem-summary/exchanges/00-elec-nem-summary.json @@ -0,0 +1,6 @@ +{ + "url_pattern": "https://visualisations.aemo.com.au/aemo/apps/api/report/ELEC_NEM_SUMMARY", + "status": 200, + "content_type": "application/json", + "body_base64": "eyJFTEVDX05FTV9TVU1NQVJZIjpbeyJTRVRUTEVNRU5UREFURSI6IjIwMjYtMDgtMTJUMTE6MDU6MDAiLCJSRUdJT05JRCI6IlFMRDEiLCJUT1RBTERFTUFORCI6NjEyMy40NSwiU0NIRURVTEVER0VORVJBVElPTiI6NTQxMC4yLCJTRU1JU0NIRURVTEVER0VORVJBVElPTiI6NDg5LjksIk5FVElOVEVSQ0hBTkdFIjotMzUwLCJQUklDRSI6ODQuNX0seyJTRVRUTEVNRU5UREFURSI6IjIwMjYtMDgtMTJUMTE6MDU6MDAiLCJSRUdJT05JRCI6Ik5TVzEiLCJUT1RBTERFTUFORCI6Nzg5MC4xMiwiU0NIRURVTEVER0VORVJBVElPTiI6Njg5MC40LCJTRU1JU0NIRURVTEVER0VORVJBVElPTiI6NjU1LjEsIk5FVElOVEVSQ0hBTkdFIjozNTAsIlBSSUNFIjo5Mi4xfSx7IlNFVFRMRU1FTlREQVRFIjoiMjAyNi0wOC0xMlQxMTowNTowMCIsIlJFR0lPTklEIjoiU0ExIiwiVE9UQUxERU1BTkQiOjE0NTAsIlNDSEVEVUxFREdFTkVSQVRJT04iOjgyMCwiU0VNSVNDSEVEVUxFREdFTkVSQVRJT04iOjUwNS41LCJORVRJTlRFUkNIQU5HRSI6MTIwLCJQUklDRSI6NDUuOH1dLCJFTEVDX05FTV9TVU1NQVJZX1BSSUNFUyI6W3siUkVHSU9OSUQiOiJRTEQxIiwiUFJJQ0UiOjg0LjUsIkNVTVVMX1BSSUNFIjoxMDIzNC4xfSx7IlJFR0lPTklEIjoiTlNXMSIsIlBSSUNFIjo5Mi4xLCJDVU1VTF9QUklDRSI6MTEwODAuNH1dLCJFTEVDX05FTV9TVU1NQVJZX01BUktFVF9OT1RJQ0UiOltdfQ==" +} diff --git a/intelligence/packages/connectors/fixtures/aisstream/README.md b/intelligence/packages/connectors/fixtures/aisstream/README.md new file mode 100644 index 0000000..52f9d3a --- /dev/null +++ b/intelligence/packages/connectors/fixtures/aisstream/README.md @@ -0,0 +1,14 @@ +# Fixtures — aisstream.io websocket + +fixture_provenance: synthetic-from-spec + +Every file in this directory is SYNTHETIC test data, generated by +`scripts/make-fixtures.mjs` from the source's published format +specification. No fixture here is real recorded data, and nothing in this +package claims live verification of the source (egress-blocked build; see +docs/LEGAL-LICENSING.md and the workspace honesty rules). + +Format spec bound against: aisstream.io documented message envelope ({MessageType, MetaData:{MMSI, ShipName, latitude, longitude, time_utc}, Message:{PositionReport|ShipStaticData}}), one JSON message per line, replayed by ReplayMessageGate with zero delay. MMSIs 503000001..503000003 are SYNTHETIC exemplar vessels (aligned with the workspace's synthetic vessel seeds); 999999999 exercises unmapped reporting. 20 messages total. + +Regenerate with: `npm run make-fixtures -w @pact-tailor/connectors` +(deterministic — fixed timestamps around 2026-08-12T01:00Z, no randomness). diff --git a/intelligence/packages/connectors/fixtures/aisstream/messages.jsonl b/intelligence/packages/connectors/fixtures/aisstream/messages.jsonl new file mode 100644 index 0000000..2165fbd --- /dev/null +++ b/intelligence/packages/connectors/fixtures/aisstream/messages.jsonl @@ -0,0 +1,20 @@ +{"MessageType":"PositionReport","MetaData":{"MMSI":503000001,"ShipName":"PACT TAILOR DEMO I ","latitude":-23.82,"longitude":151.25,"time_utc":"2026-08-12 01:00:00.000000000 +0000 UTC"},"Message":{"PositionReport":{"Cog":123.4,"Latitude":-23.82,"Longitude":151.25,"NavigationalStatus":0,"RateOfTurn":0,"Sog":8.2,"TrueHeading":121,"UserID":503000001,"Valid":true}}} +{"MessageType":"PositionReport","MetaData":{"MMSI":503000001,"ShipName":"PACT TAILOR DEMO I ","latitude":-23.819,"longitude":151.251,"time_utc":"2026-08-12 01:05:00.000000000 +0000 UTC"},"Message":{"PositionReport":{"Cog":123.4,"Latitude":-23.819,"Longitude":151.251,"NavigationalStatus":0,"RateOfTurn":0,"Sog":8.2,"TrueHeading":121,"UserID":503000001,"Valid":true}}} +{"MessageType":"PositionReport","MetaData":{"MMSI":503000001,"ShipName":"PACT TAILOR DEMO I ","latitude":-23.818,"longitude":151.252,"time_utc":"2026-08-12 01:10:00.000000000 +0000 UTC"},"Message":{"PositionReport":{"Cog":123.4,"Latitude":-23.818,"Longitude":151.252,"NavigationalStatus":0,"RateOfTurn":0,"Sog":8.2,"TrueHeading":121,"UserID":503000001,"Valid":true}}} +{"MessageType":"PositionReport","MetaData":{"MMSI":503000001,"ShipName":"PACT TAILOR DEMO I ","latitude":-23.817,"longitude":151.253,"time_utc":"2026-08-12 01:15:00.000000000 +0000 UTC"},"Message":{"PositionReport":{"Cog":123.4,"Latitude":-23.817,"Longitude":151.253,"NavigationalStatus":0,"RateOfTurn":0,"Sog":8.2,"TrueHeading":121,"UserID":503000001,"Valid":true}}} +{"MessageType":"PositionReport","MetaData":{"MMSI":503000001,"ShipName":"PACT TAILOR DEMO I ","latitude":-23.816,"longitude":151.254,"time_utc":"2026-08-12 01:20:00.000000000 +0000 UTC"},"Message":{"PositionReport":{"Cog":123.4,"Latitude":-23.816,"Longitude":151.254,"NavigationalStatus":0,"RateOfTurn":0,"Sog":8.2,"TrueHeading":121,"UserID":503000001,"Valid":true}}} +{"MessageType":"PositionReport","MetaData":{"MMSI":503000001,"ShipName":"PACT TAILOR DEMO I ","latitude":-23.815,"longitude":151.255,"time_utc":"2026-08-12 01:25:00.000000000 +0000 UTC"},"Message":{"PositionReport":{"Cog":123.4,"Latitude":-23.815,"Longitude":151.255,"NavigationalStatus":0,"RateOfTurn":0,"Sog":8.2,"TrueHeading":121,"UserID":503000001,"Valid":true}}} +{"MessageType":"PositionReport","MetaData":{"MMSI":503000002,"ShipName":"PACT TAILOR DEMO II ","latitude":-23.78,"longitude":151.18,"time_utc":"2026-08-12 01:01:00.000000000 +0000 UTC"},"Message":{"PositionReport":{"Cog":0,"Latitude":-23.78,"Longitude":151.18,"NavigationalStatus":0,"RateOfTurn":0,"Sog":0.1,"TrueHeading":511,"UserID":503000002,"Valid":true}}} +{"MessageType":"PositionReport","MetaData":{"MMSI":503000002,"ShipName":"PACT TAILOR DEMO II ","latitude":-23.779,"longitude":151.181,"time_utc":"2026-08-12 01:06:00.000000000 +0000 UTC"},"Message":{"PositionReport":{"Cog":0,"Latitude":-23.779,"Longitude":151.181,"NavigationalStatus":0,"RateOfTurn":0,"Sog":0.1,"TrueHeading":511,"UserID":503000002,"Valid":true}}} +{"MessageType":"PositionReport","MetaData":{"MMSI":503000002,"ShipName":"PACT TAILOR DEMO II ","latitude":-23.778,"longitude":151.182,"time_utc":"2026-08-12 01:11:00.000000000 +0000 UTC"},"Message":{"PositionReport":{"Cog":0,"Latitude":-23.778,"Longitude":151.182,"NavigationalStatus":0,"RateOfTurn":0,"Sog":0.1,"TrueHeading":511,"UserID":503000002,"Valid":true}}} +{"MessageType":"PositionReport","MetaData":{"MMSI":503000002,"ShipName":"PACT TAILOR DEMO II ","latitude":-23.777,"longitude":151.183,"time_utc":"2026-08-12 01:16:00.000000000 +0000 UTC"},"Message":{"PositionReport":{"Cog":0,"Latitude":-23.777,"Longitude":151.183,"NavigationalStatus":0,"RateOfTurn":0,"Sog":0.1,"TrueHeading":511,"UserID":503000002,"Valid":true}}} +{"MessageType":"PositionReport","MetaData":{"MMSI":503000002,"ShipName":"PACT TAILOR DEMO II ","latitude":-23.776,"longitude":151.184,"time_utc":"2026-08-12 01:21:00.000000000 +0000 UTC"},"Message":{"PositionReport":{"Cog":0,"Latitude":-23.776,"Longitude":151.184,"NavigationalStatus":0,"RateOfTurn":0,"Sog":0.1,"TrueHeading":511,"UserID":503000002,"Valid":true}}} +{"MessageType":"PositionReport","MetaData":{"MMSI":503000003,"ShipName":"PACT TAILOR DEMO III ","latitude":-23.95,"longitude":151.4,"time_utc":"2026-08-12 01:02:00.000000000 +0000 UTC"},"Message":{"PositionReport":{"Cog":48.2,"Latitude":-23.95,"Longitude":151.4,"NavigationalStatus":0,"RateOfTurn":0,"Sog":11.6,"TrueHeading":47,"UserID":503000003,"Valid":true}}} +{"MessageType":"PositionReport","MetaData":{"MMSI":503000003,"ShipName":"PACT TAILOR DEMO III ","latitude":-23.949,"longitude":151.401,"time_utc":"2026-08-12 01:07:00.000000000 +0000 UTC"},"Message":{"PositionReport":{"Cog":48.2,"Latitude":-23.949,"Longitude":151.401,"NavigationalStatus":0,"RateOfTurn":0,"Sog":11.6,"TrueHeading":47,"UserID":503000003,"Valid":true}}} +{"MessageType":"PositionReport","MetaData":{"MMSI":503000003,"ShipName":"PACT TAILOR DEMO III ","latitude":-23.948,"longitude":151.402,"time_utc":"2026-08-12 01:12:00.000000000 +0000 UTC"},"Message":{"PositionReport":{"Cog":48.2,"Latitude":-23.948,"Longitude":151.402,"NavigationalStatus":0,"RateOfTurn":0,"Sog":11.6,"TrueHeading":47,"UserID":503000003,"Valid":true}}} +{"MessageType":"PositionReport","MetaData":{"MMSI":503000003,"ShipName":"PACT TAILOR DEMO III ","latitude":-23.947,"longitude":151.403,"time_utc":"2026-08-12 01:17:00.000000000 +0000 UTC"},"Message":{"PositionReport":{"Cog":48.2,"Latitude":-23.947,"Longitude":151.403,"NavigationalStatus":0,"RateOfTurn":0,"Sog":11.6,"TrueHeading":47,"UserID":503000003,"Valid":true}}} +{"MessageType":"PositionReport","MetaData":{"MMSI":999999999,"ShipName":"UNSEEDED VESSEL ","latitude":-23.7,"longitude":151.3,"time_utc":"2026-08-12 01:03:00.000000000 +0000 UTC"},"Message":{"PositionReport":{"Cog":200,"Latitude":-23.7,"Longitude":151.3,"NavigationalStatus":0,"RateOfTurn":0,"Sog":5,"TrueHeading":198,"UserID":999999999,"Valid":true}}} +{"MessageType":"PositionReport","MetaData":{"MMSI":999999999,"ShipName":"UNSEEDED VESSEL ","latitude":-23.699,"longitude":151.301,"time_utc":"2026-08-12 01:08:00.000000000 +0000 UTC"},"Message":{"PositionReport":{"Cog":200,"Latitude":-23.699,"Longitude":151.301,"NavigationalStatus":0,"RateOfTurn":0,"Sog":5,"TrueHeading":198,"UserID":999999999,"Valid":true}}} +{"MessageType":"PositionReport","MetaData":{"MMSI":999999999,"ShipName":"UNSEEDED VESSEL ","latitude":-23.698,"longitude":151.302,"time_utc":"2026-08-12 01:13:00.000000000 +0000 UTC"},"Message":{"PositionReport":{"Cog":200,"Latitude":-23.698,"Longitude":151.302,"NavigationalStatus":0,"RateOfTurn":0,"Sog":5,"TrueHeading":198,"UserID":999999999,"Valid":true}}} +{"MessageType":"ShipStaticData","MetaData":{"MMSI":503000001,"ShipName":"PACT TAILOR DEMO I ","latitude":-23.82,"longitude":151.25,"time_utc":"2026-08-12 01:20:00.000000000 +0000 UTC"},"Message":{"ShipStaticData":{"CallSign":"SYNTH1","Destination":"GLADSTONE","Dimension":{"A":120,"B":80,"C":15,"D":17},"ImoNumber":0,"MaximumStaticDraught":10.5,"Name":"PACT TAILOR DEMO I","Type":70,"UserID":503000001,"Valid":true}}} +{"MessageType":"ShipStaticData","MetaData":{"MMSI":999999999,"ShipName":"UNSEEDED VESSEL ","latitude":-23.7,"longitude":151.3,"time_utc":"2026-08-12 01:21:00.000000000 +0000 UTC"},"Message":{"ShipStaticData":{"CallSign":"SYNTH1","Destination":"ELSEWHERE","Dimension":{"A":120,"B":80,"C":15,"D":17},"ImoNumber":0,"MaximumStaticDraught":7.2,"Name":"UNSEEDED VESSEL","Type":70,"UserID":999999999,"Valid":true}}} diff --git a/intelligence/packages/connectors/fixtures/bom-observations/README.md b/intelligence/packages/connectors/fixtures/bom-observations/README.md new file mode 100644 index 0000000..f9e3b14 --- /dev/null +++ b/intelligence/packages/connectors/fixtures/bom-observations/README.md @@ -0,0 +1,14 @@ +# Fixtures — BOM station observations (IDQ60801 format) + +fixture_provenance: synthetic-from-spec + +Every file in this directory is SYNTHETIC test data, generated by +`scripts/make-fixtures.mjs` from the source's published format +specification. No fixture here is real recorded data, and nothing in this +package claims live verification of the source (egress-blocked build; see +docs/LEGAL-LICENSING.md and the workspace honesty rules). + +Format spec bound against: BOM per-station JSON observation product format ({observations:{header,data:[{wmo, aifstime_utc, air_temp, wind_spd_kmh, ...}]}}, aifstime_utc in UTC yyyymmddHHMMSS). Values are invented; the station map targets the connector-owned Sensor entity sensor:au-qld:gladstone-aws. + +Regenerate with: `npm run make-fixtures -w @pact-tailor/connectors` +(deterministic — fixed timestamps around 2026-08-12T01:00Z, no randomness). diff --git a/intelligence/packages/connectors/fixtures/bom-observations/exchanges/00-idq60801-gladstone.json b/intelligence/packages/connectors/fixtures/bom-observations/exchanges/00-idq60801-gladstone.json new file mode 100644 index 0000000..3163852 --- /dev/null +++ b/intelligence/packages/connectors/fixtures/bom-observations/exchanges/00-idq60801-gladstone.json @@ -0,0 +1,6 @@ +{ + "url_pattern": "http://www.bom.gov.au/fwo/IDQ60801/IDQ60801.94381.json", + "status": 200, + "content_type": "application/json", + "body_base64": "eyJvYnNlcnZhdGlvbnMiOnsibm90aWNlIjpbeyJjb3B5cmlnaHQiOiJTWU5USEVUSUMgRklYVFVSRSDigJQgZm9ybWF0IHBlciBCT00gSlNPTiBvYnNlcnZhdGlvbiBwcm9kdWN0czsgbm90IEJPTSBkYXRhIiwiY29weXJpZ2h0X3VybCI6Imh0dHA6Ly93d3cuYm9tLmdvdi5hdS9vdGhlci9jb3B5cmlnaHQuc2h0bWwifV0sImhlYWRlciI6W3sicmVmcmVzaF9tZXNzYWdlIjoiSXNzdWVkIGF0IDExOjM1IGFtIEVTVCBXZWRuZXNkYXkgMTIgQXVndXN0IDIwMjYiLCJJRCI6IklEUTYwODAxIiwibWFpbl9JRCI6IklEUTYwODAwIiwibmFtZSI6IkdsYWRzdG9uZSBBaXJwb3J0Iiwic3RhdGVfdGltZV96b25lIjoiUUxEIiwidGltZV96b25lIjoiRVNUIiwicHJvZHVjdF9uYW1lIjoiV2VhdGhlciBPYnNlcnZhdGlvbnMiLCJzdGF0ZSI6IlF1ZWVuc2xhbmQifV0sImRhdGEiOlt7InNvcnRfb3JkZXIiOjAsIndtbyI6OTQzODEsIm5hbWUiOiJHbGFkc3RvbmUgQWlycG9ydCIsImhpc3RvcnlfcHJvZHVjdCI6IklEUTYwODAxIiwibG9jYWxfZGF0ZV90aW1lX2Z1bGwiOiIyMDI2MDgxMjExMzAwMCIsImFpZnN0aW1lX3V0YyI6IjIwMjYwODEyMDEzMDAwIiwibGF0IjotMjMuOSwibG9uIjoxNTEuMiwiYXBwYXJlbnRfdCI6MTYuOSwiY2xvdWQiOiItIiwiZ3VzdF9rbWgiOjI2LCJhaXJfdGVtcCI6MTguNCwiZGV3cHQiOjExLjIsInByZXNzIjoxMDIxLjMsInByZXNzX3FuaCI6MTAyMS4zLCJyYWluX3RyYWNlIjoiMC4wIiwicmVsX2h1bSI6NjMsInN3ZWxsX2Rpcl93b3JkZWQiOiItIiwid2luZF9kaXIiOiJTU0UiLCJ3aW5kX3NwZF9rbWgiOjE5fSx7InNvcnRfb3JkZXIiOjEsIndtbyI6OTQzODEsIm5hbWUiOiJHbGFkc3RvbmUgQWlycG9ydCIsImhpc3RvcnlfcHJvZHVjdCI6IklEUTYwODAxIiwibG9jYWxfZGF0ZV90aW1lX2Z1bGwiOiIyMDI2MDgxMjExMDAwMCIsImFpZnN0aW1lX3V0YyI6IjIwMjYwODEyMDEwMDAwIiwibGF0IjotMjMuOSwibG9uIjoxNTEuMiwiYXBwYXJlbnRfdCI6MTYuNiwiY2xvdWQiOiItIiwiZ3VzdF9rbWgiOjI0LCJhaXJfdGVtcCI6MTguMSwiZGV3cHQiOjExLjIsInByZXNzIjoxMDIxLjMsInByZXNzX3FuaCI6MTAyMS4zLCJyYWluX3RyYWNlIjoiMC4wIiwicmVsX2h1bSI6NjMsInN3ZWxsX2Rpcl93b3JkZWQiOiItIiwid2luZF9kaXIiOiJTU0UiLCJ3aW5kX3NwZF9rbWgiOjE3fSx7InNvcnRfb3JkZXIiOjIsIndtbyI6OTQzODEsIm5hbWUiOiJHbGFkc3RvbmUgQWlycG9ydCIsImhpc3RvcnlfcHJvZHVjdCI6IklEUTYwODAxIiwibG9jYWxfZGF0ZV90aW1lX2Z1bGwiOiIyMDI2MDgxMjEwMzAwMCIsImFpZnN0aW1lX3V0YyI6IjIwMjYwODEyMDAzMDAwIiwibGF0IjotMjMuOSwibG9uIjoxNTEuMiwiYXBwYXJlbnRfdCI6MTYuMywiY2xvdWQiOiItIiwiZ3VzdF9rbWgiOjIyLCJhaXJfdGVtcCI6MTcuOCwiZGV3cHQiOjExLjIsInByZXNzIjoxMDIxLjMsInByZXNzX3FuaCI6MTAyMS4zLCJyYWluX3RyYWNlIjoiMC4wIiwicmVsX2h1bSI6NjMsInN3ZWxsX2Rpcl93b3JkZWQiOiItIiwid2luZF9kaXIiOiJTU0UiLCJ3aW5kX3NwZF9rbWgiOjE1fV19fQ==" +} diff --git a/intelligence/packages/connectors/fixtures/eia-v2/README.md b/intelligence/packages/connectors/fixtures/eia-v2/README.md new file mode 100644 index 0000000..9d9423c --- /dev/null +++ b/intelligence/packages/connectors/fixtures/eia-v2/README.md @@ -0,0 +1,14 @@ +# Fixtures — EIA API v2 region-data (STUB connector) + +fixture_provenance: synthetic-from-spec + +Every file in this directory is SYNTHETIC test data, generated by +`scripts/make-fixtures.mjs` from the source's published format +specification. No fixture here is real recorded data, and nothing in this +package claims live verification of the source (egress-blocked build; see +docs/LEGAL-LICENSING.md and the workspace honesty rules). + +Format spec bound against: EIA API v2 documented response envelope ({response:{data:[{period, respondent, type, value, value-units}]}}). The live path is a stub (api_key required); only this replay fixture exercises parse/normalize. Respondent NOBA exercises unmapped reporting. + +Regenerate with: `npm run make-fixtures -w @pact-tailor/connectors` +(deterministic — fixed timestamps around 2026-08-12T01:00Z, no randomness). diff --git a/intelligence/packages/connectors/fixtures/eia-v2/exchanges/00-region-data.json b/intelligence/packages/connectors/fixtures/eia-v2/exchanges/00-region-data.json new file mode 100644 index 0000000..2d0dfe2 --- /dev/null +++ b/intelligence/packages/connectors/fixtures/eia-v2/exchanges/00-region-data.json @@ -0,0 +1,6 @@ +{ + "url_pattern": "https://api.eia.gov/v2/electricity/rto/region-data/data/", + "status": 200, + "content_type": "application/json", + "body_base64": "eyJyZXNwb25zZSI6eyJ0b3RhbCI6MywiZGF0ZUZvcm1hdCI6IllZWVktTU0tRERcIlRcIkhIMjQiLCJmcmVxdWVuY3kiOiJob3VybHkiLCJkYXRhIjpbeyJwZXJpb2QiOiIyMDI2LTA4LTEyVDAwIiwicmVzcG9uZGVudCI6IkRFTU8iLCJyZXNwb25kZW50LW5hbWUiOiJTeW50aGV0aWMgRGVtbyBCQSIsInR5cGUiOiJEIiwidHlwZS1uYW1lIjoiRGVtYW5kIiwidmFsdWUiOjIxNDUwLCJ2YWx1ZS11bml0cyI6Im1lZ2F3YXR0aG91cnMifSx7InBlcmlvZCI6IjIwMjYtMDgtMTJUMDEiLCJyZXNwb25kZW50IjoiREVNTyIsInJlc3BvbmRlbnQtbmFtZSI6IlN5bnRoZXRpYyBEZW1vIEJBIiwidHlwZSI6IkQiLCJ0eXBlLW5hbWUiOiJEZW1hbmQiLCJ2YWx1ZSI6MjA5ODAsInZhbHVlLXVuaXRzIjoibWVnYXdhdHRob3VycyJ9LHsicGVyaW9kIjoiMjAyNi0wOC0xMlQwMCIsInJlc3BvbmRlbnQiOiJOT0JBIiwicmVzcG9uZGVudC1uYW1lIjoiVW5tYXBwZWQgQkEiLCJ0eXBlIjoiRCIsInR5cGUtbmFtZSI6IkRlbWFuZCIsInZhbHVlIjo5MDAsInZhbHVlLXVuaXRzIjoibWVnYXdhdHRob3VycyJ9XX0sInJlcXVlc3QiOnsiY29tbWFuZCI6Ii92Mi9lbGVjdHJpY2l0eS9ydG8vcmVnaW9uLWRhdGEvZGF0YS8iLCJwYXJhbXMiOnsiZnJlcXVlbmN5IjoiaG91cmx5In19LCJhcGlWZXJzaW9uIjoiMi4xLjgifQ==" +} diff --git a/intelligence/packages/connectors/fixtures/entsoe-transparency/README.md b/intelligence/packages/connectors/fixtures/entsoe-transparency/README.md new file mode 100644 index 0000000..6b39872 --- /dev/null +++ b/intelligence/packages/connectors/fixtures/entsoe-transparency/README.md @@ -0,0 +1,14 @@ +# Fixtures — ENTSO-E Transparency A75 (STUB connector) + +fixture_provenance: synthetic-from-spec + +Every file in this directory is SYNTHETIC test data, generated by +`scripts/make-fixtures.mjs` from the source's published format +specification. No fixture here is real recorded data, and nothing in this +package claims live verification of the source (egress-blocked build; see +docs/LEGAL-LICENSING.md and the workspace honesty rules). + +Format spec bound against: A75 (actual generation per production type) GL_MarketDocument XML per the published Transparency Platform user guide. The live path is a stub (registration-required securityToken); only this replay fixture exercises parse/normalize. Zone 10Y-UNMAPPED---Z exercises unmapped reporting. + +Regenerate with: `npm run make-fixtures -w @pact-tailor/connectors` +(deterministic — fixed timestamps around 2026-08-12T01:00Z, no randomness). diff --git a/intelligence/packages/connectors/fixtures/entsoe-transparency/exchanges/00-a75-sample.json b/intelligence/packages/connectors/fixtures/entsoe-transparency/exchanges/00-a75-sample.json new file mode 100644 index 0000000..2656143 --- /dev/null +++ b/intelligence/packages/connectors/fixtures/entsoe-transparency/exchanges/00-a75-sample.json @@ -0,0 +1,6 @@ +{ + "url_pattern": "https://web-api.tp.entsoe.eu/api?documentType=A75&processType=A16", + "status": 200, + "content_type": "application/xml", + "body_base64": "PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPEdMX01hcmtldERvY3VtZW50IHhtbG5zPSJ1cm46aWVjNjIzMjUuMzUxOnRjNTd3ZzE2OjQ1MS02OmdlbmVyYXRpb25sb2FkZG9jdW1lbnQ6MzowIj4KICA8bVJJRD5TWU5USEVUSUNBNzVTQU1QTEUwMDAxPC9tUklEPgogIDxyZXZpc2lvbk51bWJlcj4xPC9yZXZpc2lvbk51bWJlcj4KICA8dHlwZT5BNzU8L3R5cGU+CiAgPHByb2Nlc3MucHJvY2Vzc1R5cGU+QTE2PC9wcm9jZXNzLnByb2Nlc3NUeXBlPgogIDxUaW1lU2VyaWVzPgogICAgPG1SSUQ+MTwvbVJJRD4KICAgIDxidXNpbmVzc1R5cGU+QTAxPC9idXNpbmVzc1R5cGU+CiAgICA8b2JqZWN0QWdncmVnYXRpb24+QTA4PC9vYmplY3RBZ2dyZWdhdGlvbj4KICAgIDxpbkJpZGRpbmdab25lX0RvbWFpbi5tUklEIGNvZGluZ1NjaGVtZT0iQTAxIj4xMFktVEVTVC1aT05FLS1YPC9pbkJpZGRpbmdab25lX0RvbWFpbi5tUklEPgogICAgPHF1YW50aXR5X01lYXN1cmVfVW5pdC5uYW1lPk1BVzwvcXVhbnRpdHlfTWVhc3VyZV9Vbml0Lm5hbWU+CiAgICA8Y3VydmVUeXBlPkEwMTwvY3VydmVUeXBlPgogICAgPE1rdFBTUlR5cGU+CiAgICAgIDxwc3JUeXBlPkIwNTwvcHNyVHlwZT4KICAgIDwvTWt0UFNSVHlwZT4KICAgIDxQZXJpb2Q+CiAgICAgIDx0aW1lSW50ZXJ2YWw+CiAgICAgICAgPHN0YXJ0PjIwMjYtMDgtMTJUMDA6MDBaPC9zdGFydD4KICAgICAgICA8ZW5kPjIwMjYtMDgtMTJUMDI6MDBaPC9lbmQ+CiAgICAgIDwvdGltZUludGVydmFsPgogICAgICA8cmVzb2x1dGlvbj5QVDYwTTwvcmVzb2x1dGlvbj4KICAgICAgPFBvaW50PgogICAgICAgIDxwb3NpdGlvbj4xPC9wb3NpdGlvbj4KICAgICAgICA8cXVhbnRpdHk+NjQwPC9xdWFudGl0eT4KICAgICAgPC9Qb2ludD4KICAgICAgPFBvaW50PgogICAgICAgIDxwb3NpdGlvbj4yPC9wb3NpdGlvbj4KICAgICAgICA8cXVhbnRpdHk+NjU1PC9xdWFudGl0eT4KICAgICAgPC9Qb2ludD4KICAgIDwvUGVyaW9kPgogIDwvVGltZVNlcmllcz4KICA8VGltZVNlcmllcz4KICAgIDxtUklEPjI8L21SSUQ+CiAgICA8YnVzaW5lc3NUeXBlPkEwMTwvYnVzaW5lc3NUeXBlPgogICAgPG9iamVjdEFnZ3JlZ2F0aW9uPkEwODwvb2JqZWN0QWdncmVnYXRpb24+CiAgICA8aW5CaWRkaW5nWm9uZV9Eb21haW4ubVJJRCBjb2RpbmdTY2hlbWU9IkEwMSI+MTBZLVVOTUFQUEVELS0tWjwvaW5CaWRkaW5nWm9uZV9Eb21haW4ubVJJRD4KICAgIDxxdWFudGl0eV9NZWFzdXJlX1VuaXQubmFtZT5NQVc8L3F1YW50aXR5X01lYXN1cmVfVW5pdC5uYW1lPgogICAgPGN1cnZlVHlwZT5BMDE8L2N1cnZlVHlwZT4KICAgIDxNa3RQU1JUeXBlPgogICAgICA8cHNyVHlwZT5CMTY8L3BzclR5cGU+CiAgICA8L01rdFBTUlR5cGU+CiAgICA8UGVyaW9kPgogICAgICA8dGltZUludGVydmFsPgogICAgICAgIDxzdGFydD4yMDI2LTA4LTEyVDAwOjAwWjwvc3RhcnQ+CiAgICAgICAgPGVuZD4yMDI2LTA4LTEyVDAxOjAwWjwvZW5kPgogICAgICA8L3RpbWVJbnRlcnZhbD4KICAgICAgPHJlc29sdXRpb24+UFQzME08L3Jlc29sdXRpb24+CiAgICAgIDxQb2ludD4KICAgICAgICA8cG9zaXRpb24+MTwvcG9zaXRpb24+CiAgICAgICAgPHF1YW50aXR5PjEyMDwvcXVhbnRpdHk+CiAgICAgIDwvUG9pbnQ+CiAgICAgIDxQb2ludD4KICAgICAgICA8cG9zaXRpb24+MjwvcG9zaXRpb24+CiAgICAgICAgPHF1YW50aXR5PjEzMTwvcXVhbnRpdHk+CiAgICAgIDwvUG9pbnQ+CiAgICA8L1BlcmlvZD4KICA8L1RpbWVTZXJpZXM+CjwvR0xfTWFya2V0RG9jdW1lbnQ+Cg==" +} diff --git a/intelligence/packages/connectors/package.json b/intelligence/packages/connectors/package.json index ad2dc64..e6fbc93 100644 --- a/intelligence/packages/connectors/package.json +++ b/intelligence/packages/connectors/package.json @@ -9,11 +9,16 @@ "exports": { ".": "./dist/index.js" }, + "bin": { + "pt-ingest": "dist/cli.js" + }, "engines": { "node": ">=20" }, "scripts": { - "build": "tsc -p tsconfig.json" + "build": "tsc -p tsconfig.json", + "test": "npm run build && node --test \"test/*.test.mjs\"", + "make-fixtures": "node scripts/make-fixtures.mjs" }, "dependencies": { "@pact-tailor/ontology": "*", diff --git a/intelligence/packages/connectors/scripts/make-fixtures.mjs b/intelligence/packages/connectors/scripts/make-fixtures.mjs new file mode 100644 index 0000000..ccc4476 --- /dev/null +++ b/intelligence/packages/connectors/scripts/make-fixtures.mjs @@ -0,0 +1,517 @@ +#!/usr/bin/env node +// make-fixtures.mjs — regenerates every synthetic fixture in fixtures/. +// +// DETERMINISTIC: no Date.now(), no randomness — fixed timestamps around +// 2026-08-12T01:00Z (= 11:00 NEM time). Run `npm run make-fixtures -w +// @pact-tailor/connectors` and the outputs are byte-identical. +// +// PROVENANCE: every payload below is SYNTHETIC, generated from the source's +// published format specification (fixture_provenance: synthetic-from-spec). +// No fixture contains real recorded data; MMSIs 503000001..503000003 are +// synthetic exemplar vessels, and all values are invented. + +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { zipSync } from "fflate"; + +const PKG_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const FIXTURES = join(PKG_ROOT, "fixtures"); + +// Fixed mtime for zip entries so archives are byte-stable across runs. +const FIXED_MTIME = new Date("2026-08-12T01:05:00Z"); + +function writeExchange(connectorId, filename, exchange) { + const dir = join(FIXTURES, connectorId, "exchanges"); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, filename), JSON.stringify(exchange, null, 2) + "\n"); +} + +function writeReadme(connectorId, title, specNote) { + const dir = join(FIXTURES, connectorId); + mkdirSync(dir, { recursive: true }); + const body = `# Fixtures — ${title} + +fixture_provenance: synthetic-from-spec + +Every file in this directory is SYNTHETIC test data, generated by +\`scripts/make-fixtures.mjs\` from the source's published format +specification. No fixture here is real recorded data, and nothing in this +package claims live verification of the source (egress-blocked build; see +docs/LEGAL-LICENSING.md and the workspace honesty rules). + +Format spec bound against: ${specNote} + +Regenerate with: \`npm run make-fixtures -w @pact-tailor/connectors\` +(deterministic — fixed timestamps around 2026-08-12T01:00Z, no randomness). +`; + writeFileSync(join(dir, "README.md"), body); +} + +function zipOf(entryName, text) { + const zipped = zipSync({ [entryName]: [new TextEncoder().encode(text), { mtime: FIXED_MTIME }] }, { level: 6 }); + return Buffer.from(zipped).toString("base64"); +} + +function iisListing(dirPath, names) { + const links = names + .map((n) => `${n}
`) + .join(""); + return ( + `nemweb.com.au - ${dirPath}` + + `

nemweb.com.au - ${dirPath}

` +
+    `[To Parent Directory]
${links}` +
+    `
` + ); +} + +// --------------------------------------------------------------------------- +// aemo-nemweb-dispatchis — MMS CSV per the MMS Data Model (DISPATCH package) +// --------------------------------------------------------------------------- +{ + const id = "aemo-nemweb-dispatchis"; + const base = "https://nemweb.com.au/Reports/Current/DispatchIS_Reports/"; + const older = "PUBLIC_DISPATCHIS_202608121100_0000000123456700.zip"; + const newest = "PUBLIC_DISPATCHIS_202608121105_0000000123456789.zip"; + // NEM time 2026/08/12 11:05:00 == 2026-08-12T01:05:00Z. + const csv = [ + 'C,NEMP.WORLD,DISPATCHIS,AEMO,PUBLIC,2026/08/12,11:05:04,0000000123456789,DISPATCHIS,0000000123456788', + 'I,DISPATCH,PRICE,4,SETTLEMENTDATE,RUNNO,REGIONID,DISPATCHINTERVAL,INTERVENTION,RRP,EEP,ROP,APCFLAG', + 'D,DISPATCH,PRICE,4,"2026/08/12 11:05:00",1,QLD1,20260812133,0,84.50,0,84.50,0', + 'D,DISPATCH,PRICE,4,"2026/08/12 11:05:00",1,NSW1,20260812133,0,92.10,0,92.10,0', + 'D,DISPATCH,PRICE,4,"2026/08/12 11:05:00",2,NSW1,20260812133,1,95.75,0,95.75,0', + 'I,DISPATCH,REGIONSUM,4,SETTLEMENTDATE,RUNNO,REGIONID,DISPATCHINTERVAL,INTERVENTION,TOTALDEMAND,AVAILABLEGENERATION,DISPATCHABLEGENERATION,NETINTERCHANGE', + 'D,DISPATCH,REGIONSUM,4,"2026/08/12 11:05:00",1,QLD1,20260812133,0,6123.45,8200.00,5900.10,-350.00', + 'D,DISPATCH,REGIONSUM,4,"2026/08/12 11:05:00",1,NSW1,20260812133,0,7890.12,9100.00,7550.00,350.00', + 'D,DISPATCH,REGIONSUM,4,"2026/08/12 11:05:00",1,SA1,20260812133,0,1450.00,2100.00,1400.00,120.00', + 'I,DISPATCH,INTERCONNECTORRES,3,SETTLEMENTDATE,RUNNO,INTERCONNECTORID,DISPATCHINTERVAL,INTERVENTION,METEREDMWFLOW,MWFLOW,MWLOSSES', + 'D,DISPATCH,INTERCONNECTORRES,3,"2026/08/12 11:05:00",1,NSW1-QLD1,20260812133,0,348.20,350.00,12.40', + 'D,DISPATCH,INTERCONNECTORRES,3,"2026/08/12 11:05:00",1,MADEUP-X1,20260812133,0,10.00,10.00,0.10', + 'C,"END OF REPORT",13', + '', + ].join("\r\n"); + writeExchange(id, "00-dir-listing.json", { + url_pattern: base, + status: 200, + content_type: "text/html", + body_base64: Buffer.from(iisListing("/Reports/Current/DispatchIS_Reports/", [older, newest])).toString("base64"), + }); + writeExchange(id, "01-dispatchis-zip.json", { + url_pattern: base + newest, + status: 200, + content_type: "application/x-zip-compressed", + body_base64: zipOf(newest.replace(/\.zip$/, ".CSV"), csv), + }); + writeReadme( + id, + "AEMO NEMWEB DispatchIS", + "AEMO MMS Data Model report CSV (C/I/D rows), tables DISPATCH.PRICE / DISPATCH.REGIONSUM / DISPATCH.INTERCONNECTORRES, in a single-entry ZIP behind an IIS-style directory listing. The NSW1 RUNNO=2/INTERVENTION=1 PRICE row exercises correction marking; SA1 and MADEUP-X1 exercise unmapped reporting (against the unit tests' inline entity index).", + ); +} + +// --------------------------------------------------------------------------- +// aemo-nemweb-dispatch-scada — DISPATCH.UNIT_SCADA +// --------------------------------------------------------------------------- +{ + const id = "aemo-nemweb-dispatch-scada"; + const base = "https://nemweb.com.au/Reports/Current/Dispatch_SCADA/"; + const newest = "PUBLIC_DISPATCHSCADA_202608121105_0000000123456791.zip"; + const units = [ + ["GSTONE1", "231.50"], + ["GSTONE2", "228.90"], + ["GSTONE3", "0.00"], + ["GSTONE4", "235.10"], + ["GSTONE5", "242.75"], + ["GSTONE6", "238.60"], + ["CALL_B_1", "312.40"], + ["NOMAP_1", "55.00"], + ]; + const csv = [ + 'C,NEMP.WORLD,DISPATCHSCADA,AEMO,PUBLIC,2026/08/12,11:05:03,0000000123456791,DISPATCHSCADA,0000000123456790', + 'I,DISPATCH,UNIT_SCADA,1,SETTLEMENTDATE,DUID,SCADAVALUE', + ...units.map(([duid, mw]) => `D,DISPATCH,UNIT_SCADA,1,"2026/08/12 11:05:00",${duid},${mw}`), + 'C,"END OF REPORT",10', + '', + ].join("\r\n"); + writeExchange(id, "00-dir-listing.json", { + url_pattern: base, + status: 200, + content_type: "text/html", + body_base64: Buffer.from(iisListing("/Reports/Current/Dispatch_SCADA/", [newest])).toString("base64"), + }); + writeExchange(id, "01-dispatchscada-zip.json", { + url_pattern: base + newest, + status: 200, + content_type: "application/x-zip-compressed", + body_base64: zipOf(newest.replace(/\.zip$/, ".CSV"), csv), + }); + writeReadme( + id, + "AEMO NEMWEB Dispatch_SCADA", + "AEMO MMS Data Model report CSV, table DISPATCH.UNIT_SCADA (SETTLEMENTDATE, DUID, SCADAVALUE) in a single-entry ZIP. GSTONE1-6 and CALL_B_1 map to seed generators; NOMAP_1 exercises unmapped reporting.", + ); +} + +// --------------------------------------------------------------------------- +// aemo-nemweb-tradingis — TRADING.PRICE +// --------------------------------------------------------------------------- +{ + const id = "aemo-nemweb-tradingis"; + const base = "https://nemweb.com.au/Reports/Current/TradingIS_Reports/"; + const newest = "PUBLIC_TRADINGIS_202608121105_0000000123456792.zip"; + const csv = [ + 'C,NEMP.WORLD,TRADINGIS,AEMO,PUBLIC,2026/08/12,11:05:05,0000000123456792,TRADINGIS,0000000123456791', + 'I,TRADING,PRICE,3,SETTLEMENTDATE,RUNNO,REGIONID,PERIODID,RRP,EEP,INVALIDFLAG', + 'D,TRADING,PRICE,3,"2026/08/12 11:05:00",1,QLD1,17,86.20,0,0', + 'D,TRADING,PRICE,3,"2026/08/12 11:05:00",1,NSW1,17,93.40,0,0', + 'D,TRADING,PRICE,3,"2026/08/12 11:05:00",1,SA1,17,71.90,0,0', + 'C,"END OF REPORT",7', + '', + ].join("\r\n"); + writeExchange(id, "00-dir-listing.json", { + url_pattern: base, + status: 200, + content_type: "text/html", + body_base64: Buffer.from(iisListing("/Reports/Current/TradingIS_Reports/", [newest])).toString("base64"), + }); + writeExchange(id, "01-tradingis-zip.json", { + url_pattern: base + newest, + status: 200, + content_type: "application/x-zip-compressed", + body_base64: zipOf(newest.replace(/\.zip$/, ".CSV"), csv), + }); + writeReadme( + id, + "AEMO NEMWEB TradingIS", + "AEMO MMS Data Model report CSV, table TRADING.PRICE (trading-interval RRP per region) in a single-entry ZIP. SA1 exercises unmapped reporting against the unit tests' inline index.", + ); +} + +// --------------------------------------------------------------------------- +// aemo-vis-nem-summary — dashboard JSON (documented-from-public-dashboard) +// --------------------------------------------------------------------------- +{ + const id = "aemo-vis-nem-summary"; + const body = { + ELEC_NEM_SUMMARY: [ + { + SETTLEMENTDATE: "2026-08-12T11:05:00", + REGIONID: "QLD1", + TOTALDEMAND: 6123.45, + SCHEDULEDGENERATION: 5410.2, + SEMISCHEDULEDGENERATION: 489.9, + NETINTERCHANGE: -350.0, + PRICE: 84.5, + }, + { + SETTLEMENTDATE: "2026-08-12T11:05:00", + REGIONID: "NSW1", + TOTALDEMAND: 7890.12, + SCHEDULEDGENERATION: 6890.4, + SEMISCHEDULEDGENERATION: 655.1, + NETINTERCHANGE: 350.0, + PRICE: 92.1, + }, + { + SETTLEMENTDATE: "2026-08-12T11:05:00", + REGIONID: "SA1", + TOTALDEMAND: 1450.0, + SCHEDULEDGENERATION: 820.0, + SEMISCHEDULEDGENERATION: 505.5, + NETINTERCHANGE: 120.0, + PRICE: 45.8, + }, + ], + ELEC_NEM_SUMMARY_PRICES: [ + { REGIONID: "QLD1", PRICE: 84.5, CUMUL_PRICE: 10234.1 }, + { REGIONID: "NSW1", PRICE: 92.1, CUMUL_PRICE: 11080.4 }, + ], + ELEC_NEM_SUMMARY_MARKET_NOTICE: [], + }; + writeExchange(id, "00-elec-nem-summary.json", { + url_pattern: "https://visualisations.aemo.com.au/aemo/apps/api/report/ELEC_NEM_SUMMARY", + status: 200, + content_type: "application/json", + body_base64: Buffer.from(JSON.stringify(body)).toString("base64"), + }); + writeReadme( + id, + "AEMO visualisations ELEC_NEM_SUMMARY", + "JSON shape of AEMO's public NEM data dashboard backing endpoint (documented-from-public-dashboard; no formal API spec — see the connector's header comment and the source registry record). SA1 exercises unmapped reporting against the unit tests' inline index.", + ); +} + +// --------------------------------------------------------------------------- +// aisstream — websocket messages JSONL (documented at aisstream.io) +// --------------------------------------------------------------------------- +{ + const id = "aisstream"; + const vessels = [ + { mmsi: 503000001, name: "PACT TAILOR DEMO I", lat: -23.82, lon: 151.25, sog: 8.2, cog: 123.4, hdg: 121 }, + { mmsi: 503000002, name: "PACT TAILOR DEMO II", lat: -23.78, lon: 151.18, sog: 0.1, cog: 0.0, hdg: 511 }, + { mmsi: 503000003, name: "PACT TAILOR DEMO III", lat: -23.95, lon: 151.4, sog: 11.6, cog: 48.2, hdg: 47 }, + { mmsi: 999999999, name: "UNSEEDED VESSEL", lat: -23.7, lon: 151.3, sog: 5.0, cog: 200.0, hdg: 198 }, + ]; + const t = (minute) => + `2026-08-12 01:${String(minute).padStart(2, "0")}:00.000000000 +0000 UTC`; + const positionReport = (v, minute, step) => ({ + MessageType: "PositionReport", + MetaData: { + MMSI: v.mmsi, + ShipName: `${v.name} `, + latitude: Number((v.lat + step * 0.001).toFixed(4)), + longitude: Number((v.lon + step * 0.001).toFixed(4)), + time_utc: t(minute), + }, + Message: { + PositionReport: { + Cog: v.cog, + Latitude: Number((v.lat + step * 0.001).toFixed(4)), + Longitude: Number((v.lon + step * 0.001).toFixed(4)), + NavigationalStatus: 0, + RateOfTurn: 0, + Sog: v.sog, + TrueHeading: v.hdg, + UserID: v.mmsi, + Valid: true, + }, + }, + }); + const staticData = (v, minute, draught, dest) => ({ + MessageType: "ShipStaticData", + MetaData: { MMSI: v.mmsi, ShipName: `${v.name} `, latitude: v.lat, longitude: v.lon, time_utc: t(minute) }, + Message: { + ShipStaticData: { + CallSign: "SYNTH1", + Destination: dest, + Dimension: { A: 120, B: 80, C: 15, D: 17 }, + ImoNumber: 0, + MaximumStaticDraught: draught, + Name: v.name, + Type: 70, + UserID: v.mmsi, + Valid: true, + }, + }, + }); + const messages = []; + // 6 position reports for DEMO I, 5 for DEMO II, 4 for DEMO III, 3 unmapped. + for (let i = 0; i < 6; i++) messages.push(positionReport(vessels[0], i * 5, i)); + for (let i = 0; i < 5; i++) messages.push(positionReport(vessels[1], 1 + i * 5, i)); + for (let i = 0; i < 4; i++) messages.push(positionReport(vessels[2], 2 + i * 5, i)); + for (let i = 0; i < 3; i++) messages.push(positionReport(vessels[3], 3 + i * 5, i)); + // 2 static-data messages: one mapped (draught observed), one unmapped. + messages.push(staticData(vessels[0], 20, 10.5, "GLADSTONE")); + messages.push(staticData(vessels[3], 21, 7.2, "ELSEWHERE")); + const dir = join(FIXTURES, id); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, "messages.jsonl"), messages.map((m) => JSON.stringify(m)).join("\n") + "\n"); + writeReadme( + id, + "aisstream.io websocket", + "aisstream.io documented message envelope ({MessageType, MetaData:{MMSI, ShipName, latitude, longitude, time_utc}, Message:{PositionReport|ShipStaticData}}), one JSON message per line, replayed by ReplayMessageGate with zero delay. MMSIs 503000001..503000003 are SYNTHETIC exemplar vessels (aligned with the workspace's synthetic vessel seeds); 999999999 exercises unmapped reporting. 20 messages total.", + ); +} + +// --------------------------------------------------------------------------- +// bom-observations — IDQ60801-format station JSON +// --------------------------------------------------------------------------- +{ + const id = "bom-observations"; + const row = (sortOrder, aifs, localFull, temp, windKmh) => ({ + sort_order: sortOrder, + wmo: 94381, + name: "Gladstone Airport", + history_product: "IDQ60801", + local_date_time_full: localFull, + aifstime_utc: aifs, + lat: -23.9, + lon: 151.2, + apparent_t: temp - 1.5, + cloud: "-", + gust_kmh: windKmh + 7, + air_temp: temp, + dewpt: 11.2, + press: 1021.3, + press_qnh: 1021.3, + rain_trace: "0.0", + rel_hum: 63, + swell_dir_worded: "-", + wind_dir: "SSE", + wind_spd_kmh: windKmh, + }); + const body = { + observations: { + notice: [ + { + copyright: "SYNTHETIC FIXTURE — format per BOM JSON observation products; not BOM data", + copyright_url: "http://www.bom.gov.au/other/copyright.shtml", + }, + ], + header: [ + { + refresh_message: "Issued at 11:35 am EST Wednesday 12 August 2026", + ID: "IDQ60801", + main_ID: "IDQ60800", + name: "Gladstone Airport", + state_time_zone: "QLD", + time_zone: "EST", + product_name: "Weather Observations", + state: "Queensland", + }, + ], + data: [ + row(0, "20260812013000", "20260812113000", 18.4, 19), + row(1, "20260812010000", "20260812110000", 18.1, 17), + row(2, "20260812003000", "20260812103000", 17.8, 15), + ], + }, + }; + writeExchange(id, "00-idq60801-gladstone.json", { + url_pattern: "http://www.bom.gov.au/fwo/IDQ60801/IDQ60801.94381.json", + status: 200, + content_type: "application/json", + body_base64: Buffer.from(JSON.stringify(body)).toString("base64"), + }); + writeReadme( + id, + "BOM station observations (IDQ60801 format)", + "BOM per-station JSON observation product format ({observations:{header,data:[{wmo, aifstime_utc, air_temp, wind_spd_kmh, ...}]}}, aifstime_utc in UTC yyyymmddHHMMSS). Values are invented; the station map targets the connector-owned Sensor entity sensor:au-qld:gladstone-aws.", + ); +} + +// --------------------------------------------------------------------------- +// entsoe-transparency — A75 GL_MarketDocument XML sample +// --------------------------------------------------------------------------- +{ + const id = "entsoe-transparency"; + const xml = ` + + SYNTHETICA75SAMPLE0001 + 1 + A75 + A16 + + 1 + A01 + A08 + 10Y-TEST-ZONE--X + MAW + A01 + + B05 + + + + 2026-08-12T00:00Z + 2026-08-12T02:00Z + + PT60M + + 1 + 640 + + + 2 + 655 + + + + + 2 + A01 + A08 + 10Y-UNMAPPED---Z + MAW + A01 + + B16 + + + + 2026-08-12T00:00Z + 2026-08-12T01:00Z + + PT30M + + 1 + 120 + + + 2 + 131 + + + + +`; + writeExchange(id, "00-a75-sample.json", { + url_pattern: "https://web-api.tp.entsoe.eu/api?documentType=A75&processType=A16", + status: 200, + content_type: "application/xml", + body_base64: Buffer.from(xml).toString("base64"), + }); + writeReadme( + id, + "ENTSO-E Transparency A75 (STUB connector)", + "A75 (actual generation per production type) GL_MarketDocument XML per the published Transparency Platform user guide. The live path is a stub (registration-required securityToken); only this replay fixture exercises parse/normalize. Zone 10Y-UNMAPPED---Z exercises unmapped reporting.", + ); +} + +// --------------------------------------------------------------------------- +// eia-v2 — API v2 region-data JSON +// --------------------------------------------------------------------------- +{ + const id = "eia-v2"; + const body = { + response: { + total: 3, + dateFormat: 'YYYY-MM-DD"T"HH24', + frequency: "hourly", + data: [ + { + period: "2026-08-12T00", + respondent: "DEMO", + "respondent-name": "Synthetic Demo BA", + type: "D", + "type-name": "Demand", + value: 21450, + "value-units": "megawatthours", + }, + { + period: "2026-08-12T01", + respondent: "DEMO", + "respondent-name": "Synthetic Demo BA", + type: "D", + "type-name": "Demand", + value: 20980, + "value-units": "megawatthours", + }, + { + period: "2026-08-12T00", + respondent: "NOBA", + "respondent-name": "Unmapped BA", + type: "D", + "type-name": "Demand", + value: 900, + "value-units": "megawatthours", + }, + ], + }, + request: { command: "/v2/electricity/rto/region-data/data/", params: { frequency: "hourly" } }, + apiVersion: "2.1.8", + }; + writeExchange(id, "00-region-data.json", { + url_pattern: "https://api.eia.gov/v2/electricity/rto/region-data/data/", + status: 200, + content_type: "application/json", + body_base64: Buffer.from(JSON.stringify(body)).toString("base64"), + }); + writeReadme( + id, + "EIA API v2 region-data (STUB connector)", + "EIA API v2 documented response envelope ({response:{data:[{period, respondent, type, value, value-units}]}}). The live path is a stub (api_key required); only this replay fixture exercises parse/normalize. Respondent NOBA exercises unmapped reporting.", + ); +} + +console.log("fixtures regenerated under", FIXTURES); diff --git a/intelligence/packages/connectors/src/aemo-csv.ts b/intelligence/packages/connectors/src/aemo-csv.ts new file mode 100644 index 0000000..9bace69 --- /dev/null +++ b/intelligence/packages/connectors/src/aemo-csv.ts @@ -0,0 +1,110 @@ +// AEMO MMS CSV report format parser (documented in the MMS Data Model report +// specifications). A report is a CSV where the first field of each line is a +// row type: +// +// C comment/header rows, including the trailing `C,"END OF REPORT",` +// I column-header row: I,,,,,,... +// D data row matching the most recent I row for its GROUP/TABLE: +// D,,
,,,,... +// +// Fields may be double-quoted (timestamps like "2026/08/12 11:05:00" always +// are); quotes escape embedded commas and doubled quotes (""). + +export interface AemoTable { + /** "GROUP.TABLE", e.g. "DISPATCH.REGIONSUM". */ + table: string; + rows: Record[]; +} + +/** Split one CSV line, honouring double-quoted fields and "" escapes. */ +export function splitCsvLine(line: string): string[] { + const fields: string[] = []; + let current = ""; + let inQuotes = false; + for (let i = 0; i < line.length; i++) { + const ch = line[i]; + if (inQuotes) { + if (ch === '"') { + if (line[i + 1] === '"') { + current += '"'; + i++; + } else { + inQuotes = false; + } + } else { + current += ch; + } + } else if (ch === '"') { + inQuotes = true; + } else if (ch === ",") { + fields.push(current); + current = ""; + } else { + current += ch; + } + } + fields.push(current); + return fields; +} + +/** + * Parse a full AEMO MMS CSV report into its tables. Repeated I rows for the + * same GROUP.TABLE merge their D rows into one table (column set follows the + * most recent I row). D rows arriving before any I row for their table are an + * error — the format guarantees I precedes D. + */ +export function parseAemoCsv(text: string): AemoTable[] { + const columnsByTable = new Map(); + const tables = new Map(); + + for (const rawLine of text.split(/\r?\n/)) { + const line = rawLine.trim(); + if (line.length === 0) continue; + const fields = splitCsvLine(line); + const rowType = fields[0]; + if (rowType === "C") continue; // headers and "END OF REPORT" + if (rowType !== "I" && rowType !== "D") continue; // unknown row types skipped + const group = fields[1] ?? ""; + const tableName = fields[2] ?? ""; + const key = `${group}.${tableName}`; + if (rowType === "I") { + columnsByTable.set(key, fields.slice(4)); + if (!tables.has(key)) tables.set(key, { table: key, rows: [] }); + continue; + } + const columns = columnsByTable.get(key); + if (columns === undefined) { + throw new Error(`aemo-csv: D row for ${key} before its I row`); + } + const row: Record = {}; + const values = fields.slice(4); + columns.forEach((col, i) => { + row[col] = values[i] ?? ""; + }); + tables.get(key)?.rows.push(row); + } + return [...tables.values()]; +} + +// --------------------------------------------------------------------------- +// NEM time +// --------------------------------------------------------------------------- + +// AEMO NEM market time is Australian Eastern Standard Time, fixed UTC+10 with +// NO daylight saving — so the conversion is a constant 10-hour subtraction. +const NEM_UTC_OFFSET_MS = 10 * 3600 * 1000; + +const NEM_TIME_RE = /^(\d{4})[/-](\d{2})[/-](\d{2})[ T](\d{2}):(\d{2}):(\d{2})$/; + +/** + * Convert a NEM timestamp ("2026/08/12 14:05:00", also the dashboard's + * "2026-08-12T14:05:00" variant) to ISO-8601 UTC by subtracting 10 hours. + */ +export function nemTimeToIso(nemTime: string): string { + const m = NEM_TIME_RE.exec(nemTime.trim()); + if (!m) throw new Error(`aemo-csv: unrecognised NEM timestamp "${nemTime}"`); + const [, y, mo, d, h, mi, s] = m; + const utcMs = + Date.UTC(Number(y), Number(mo) - 1, Number(d), Number(h), Number(mi), Number(s)) - NEM_UTC_OFFSET_MS; + return new Date(utcMs).toISOString().replace(".000Z", "Z"); +} diff --git a/intelligence/packages/connectors/src/cli.ts b/intelligence/packages/connectors/src/cli.ts new file mode 100644 index 0000000..4a14dcc --- /dev/null +++ b/intelligence/packages/connectors/src/cli.ts @@ -0,0 +1,178 @@ +#!/usr/bin/env node +// pt-ingest — connector ingestion CLI. +// +// pt-ingest ingest --replay [--var ] [--data ] +// Run every connector once against its committed fixtures via +// ReplayGate/ReplayMessageGate (synthetic-from-spec — no network). +// Prints a per-connector IngestReport table. Exit 0 even with unmapped +// ids (they are reported, not errors); non-zero on hard errors. +// +// pt-ingest ingest --watch [--var ] [--data ] +// LiveGate + cooperative scheduler loop. In an egress-blocked +// environment this fails fast per pass — expected and honest. + +import { existsSync, mkdirSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { loadRegistry } from "@pact-tailor/registry"; +import { MedallionStore } from "@pact-tailor/store"; +import type { EntityRecord } from "@pact-tailor/ontology"; +import { EntityIndex } from "./entity-index.js"; +import { LiveGate, LiveMessageGate, ReplayGate, ReplayMessageGate } from "./gate.js"; +import type { Connector, ConnectorCtx } from "./sdk.js"; +import { runOnce, Scheduler, type IngestReport } from "./scheduler.js"; +import { allConnectors } from "./connectors/index.js"; + +const PKG_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); // packages/connectors +const INTEL_ROOT = resolve(PKG_ROOT, "..", ".."); // intelligence/ + +interface CliArgs { + command: string | undefined; + replay: boolean; + watch: boolean; + varDir: string; + dataDir: string; +} + +function parseArgs(argv: string[]): CliArgs { + const args: CliArgs = { + command: undefined, + replay: false, + watch: false, + varDir: join(INTEL_ROOT, "var"), + dataDir: join(INTEL_ROOT, "data"), + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === undefined) continue; + if (arg === "--replay") args.replay = true; + else if (arg === "--watch") args.watch = true; + else if (arg === "--var") { + const v = argv[++i]; + if (v === undefined) throw new Error("--var requires a directory argument"); + args.varDir = resolve(v); + } else if (arg === "--data") { + const v = argv[++i]; + if (v === undefined) throw new Error("--data requires a directory argument"); + args.dataDir = resolve(v); + } else if (!arg.startsWith("-") && args.command === undefined) args.command = arg; + else throw new Error(`unknown argument: ${arg}`); + } + return args; +} + +/** Registry entities + connector seedEntities (registry wins on collision). */ +function buildEntityIndex(dataDir: string, connectors: Connector[]): EntityIndex { + const byId = new Map(); + for (const connector of connectors) { + for (const seed of connector.seedEntities ?? []) byId.set(seed.entity_id, seed); + } + if (existsSync(dataDir)) { + const { data, errors } = loadRegistry(dataDir); + if (errors.length > 0) { + console.error(`warning: registry load reported ${errors.length} validation error(s); invalid records excluded`); + } + for (const entity of data.entities) byId.set(entity.entity_id, entity); + } else { + console.error(`warning: data dir ${dataDir} does not exist; EntityIndex holds connector seed entities only`); + } + return new EntityIndex([...byId.values()]); +} + +function makeCtxFactory(mode: "replay" | "live", entityIndex: EntityIndex): (connector: Connector) => ConnectorCtx { + return (connector: Connector): ConnectorCtx => { + const log = (msg: string) => console.error(`[${connector.id}] ${msg}`); + if (mode === "replay") { + const fixturesDir = join(PKG_ROOT, "fixtures", connector.id); + return { + gate: new ReplayGate(fixturesDir), + messageGate: new ReplayMessageGate(join(fixturesDir, "messages.jsonl")), + now: () => new Date().toISOString(), + entityIndex, + log, + }; + } + return { + gate: new LiveGate(), + messageGate: new LiveMessageGate(), + now: () => new Date().toISOString(), + entityIndex, + log, + }; + }; +} + +function printReportTable(reports: IngestReport[]): void { + const headers = ["connector", "artifacts", "skipped", "observations", "unmapped", "errors"]; + const rows = reports.map((r) => [ + r.connector_id, + String(r.artifacts), + String(r.skipped), + String(r.observations), + r.unmapped.length === 0 ? "-" : `${r.unmapped.length}: ${r.unmapped.join(",")}`, + r.errors.length === 0 ? "-" : `${r.errors.length}: ${r.errors.join(" | ")}`, + ]); + const widths = headers.map((h, i) => Math.max(h.length, ...rows.map((row) => (row[i] ?? "").length))); + const line = (cells: string[]) => cells.map((c, i) => c.padEnd(widths[i] ?? 0)).join(" "); + console.log(line(headers)); + console.log(line(widths.map((w) => "-".repeat(w)))); + for (const row of rows) console.log(line(row)); +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + if (args.command !== "ingest" || args.replay === args.watch) { + console.error("usage: pt-ingest ingest (--replay | --watch) [--var ] [--data ]"); + return 2; + } + + mkdirSync(args.varDir, { recursive: true }); + const store = new MedallionStore(args.varDir); + const entityIndex = buildEntityIndex(args.dataDir, allConnectors); + console.error(`entity index: ${entityIndex.size} entities (registry + connector seeds)`); + + if (args.replay) { + const makeCtx = makeCtxFactory("replay", entityIndex); + const reports: IngestReport[] = []; + for (const connector of allConnectors) { + reports.push(await runOnce(connector, makeCtx(connector), store)); + } + printReportTable(reports); + const hardErrors = reports.flatMap((r) => r.errors); + if (hardErrors.length > 0) { + console.error(`${hardErrors.length} hard error(s) — see table`); + return 1; + } + return 0; + } + + // --watch: live scheduler loop. Runs until SIGINT/SIGTERM. Under blocked + // egress every pass fails fast — that is expected, and reported honestly. + const makeCtx = makeCtxFactory("live", entityIndex); + const scheduler = new Scheduler(store, makeCtx, (report) => { + const status = report.errors.length > 0 ? "ERR" : "ok"; + console.log( + `[${new Date().toISOString()}] ${report.connector_id}: ${status} artifacts=${report.artifacts} skipped=${report.skipped} obs=${report.observations}` + + (report.unmapped.length > 0 ? ` unmapped=${report.unmapped.join(",")}` : "") + + (report.errors.length > 0 ? ` errors=${report.errors.join(" | ")}` : ""), + ); + }); + scheduler.start(allConnectors); + await new Promise((resolveWait) => { + const stop = () => { + scheduler.stop(); + resolveWait(); + }; + process.on("SIGINT", stop); + process.on("SIGTERM", stop); + }); + return 0; +} + +main().then( + (code) => process.exit(code), + (err) => { + console.error(`pt-ingest: ${(err as Error).message}`); + process.exit(1); + }, +); diff --git a/intelligence/packages/connectors/src/connectors/aemo-nemweb-dispatch-scada.ts b/intelligence/packages/connectors/src/connectors/aemo-nemweb-dispatch-scada.ts new file mode 100644 index 0000000..4a76edc --- /dev/null +++ b/intelligence/packages/connectors/src/connectors/aemo-nemweb-dispatch-scada.ts @@ -0,0 +1,76 @@ +// aemo-nemweb-dispatch-scada — NEMWEB Current Dispatch_SCADA (unit-level +// SCADA MW per dispatch interval). Wire format bound from the published MMS +// Data Model: table DISPATCH.UNIT_SCADA (SETTLEMENTDATE, DUID, SCADAVALUE). +// Built against the documented format only — fixtures are synthetic-from-spec. + +import type { TelemetryFeedRecord } from "@pact-tailor/ontology"; +import type { Connector, ConnectorCtx, NormalizedOutput, ParsedRecord } from "../sdk.js"; +import { nemTimeToIso } from "../aemo-csv.js"; +import { fieldNum, fieldStr, makeObservation, reportUnmapped } from "../observations.js"; +import { makeNemwebDiscover, nemwebFetch, nemwebParseZip } from "./nemweb-common.js"; + +const CONNECTOR_ID = "aemo-nemweb-dispatch-scada"; +const SOURCE_ID = "source:aemo:nemweb-dispatch-scada"; +const BASE_URL = "https://nemweb.com.au/Reports/Current/Dispatch_SCADA/"; +const FILENAME_RE = /PUBLIC_DISPATCHSCADA_\d{12}_\d+\.zip/; + +const FEED_SCADA = "feed:au-nem:dispatch:scada"; + +const feeds: TelemetryFeedRecord[] = [ + { + feed_id: FEED_SCADA, + source_id: SOURCE_ID, + connector_id: CONNECTOR_ID, + metrics: ["power.output.mw"], + cadence: "5min", + description: "DISPATCH.UNIT_SCADA — instantaneous unit output (SCADAVALUE) per DUID", + entity_scope: "Generator/StorageAsset units (external_ids.aemo_duid)", + }, +]; + +function normalize(records: ParsedRecord[], ctx: ConnectorCtx): NormalizedOutput { + const observations: NormalizedOutput["observations"] = []; + const unmapped: string[] = []; + const ingestTime = ctx.now(); + + for (const { table, row } of records) { + if (table !== "DISPATCH.UNIT_SCADA") continue; + const duid = fieldStr(row, "DUID"); + const entityId = ctx.entityIndex.byDuid(duid); + if (entityId === undefined) { + reportUnmapped(unmapped, duid); + continue; + } + const value = fieldNum(row, "SCADAVALUE"); + if (value === null) continue; + const settlement = fieldStr(row, "SETTLEMENTDATE"); + observations.push( + makeObservation({ + entity_id: entityId, + metric: "power.output.mw", + value, + unit: "MW", + event_time: nemTimeToIso(settlement), + ingest_time: ingestTime, + source_id: SOURCE_ID, + feed_id: FEED_SCADA, + source_sequence: settlement, + // A station can map several DUIDs to one entity; keep the unit id so + // downstream aggregation can tell the units apart. + meta: { duid }, + }), + ); + } + return { observations, unmapped }; +} + +export const aemoNemwebDispatchScada: Connector = { + id: CONNECTOR_ID, + source_id: SOURCE_ID, + feeds, + schedule: { intervalMs: 5 * 60_000, jitterMs: 30_000 }, + discover: makeNemwebDiscover(BASE_URL, FILENAME_RE), + fetch: nemwebFetch, + parse: nemwebParseZip, + normalize, +}; diff --git a/intelligence/packages/connectors/src/connectors/aemo-nemweb-dispatchis.ts b/intelligence/packages/connectors/src/connectors/aemo-nemweb-dispatchis.ts new file mode 100644 index 0000000..64b705e --- /dev/null +++ b/intelligence/packages/connectors/src/connectors/aemo-nemweb-dispatchis.ts @@ -0,0 +1,149 @@ +// aemo-nemweb-dispatchis — NEMWEB Current DispatchIS_Reports (5-minute +// dispatch solution). Wire format bound from the published MMS Data Model +// (DISPATCH package): tables DISPATCH.REGIONSUM (regional demand/generation), +// DISPATCH.PRICE (regional RRP), DISPATCH.INTERCONNECTORRES (signed MWFLOW). +// Built against the documented format only — fixtures are synthetic-from-spec. + +import type { TelemetryFeedRecord } from "@pact-tailor/ontology"; +import type { Connector, ConnectorCtx, NormalizedOutput, ParsedRecord } from "../sdk.js"; +import { nemTimeToIso } from "../aemo-csv.js"; +import { fieldNum, fieldStr, makeObservation, reportUnmapped } from "../observations.js"; +import { CorrectionTracker, makeNemwebDiscover, nemwebFetch, nemwebParseZip } from "./nemweb-common.js"; + +const CONNECTOR_ID = "aemo-nemweb-dispatchis"; +const SOURCE_ID = "source:aemo:nemweb-dispatchis"; +const BASE_URL = "https://nemweb.com.au/Reports/Current/DispatchIS_Reports/"; +const FILENAME_RE = /PUBLIC_DISPATCHIS_\d{12}_\d+\.zip/; + +const FEED_REGIONSUM = "feed:au-nem:dispatch:regionsum"; +const FEED_PRICE = "feed:au-nem:dispatch:price"; +const FEED_INTERCONNECTOR = "feed:au-nem:dispatch:interconnectorres"; + +const feeds: TelemetryFeedRecord[] = [ + { + feed_id: FEED_REGIONSUM, + source_id: SOURCE_ID, + connector_id: CONNECTOR_ID, + metrics: ["grid.demand.mw", "grid.generation.mw"], + cadence: "5min", + description: "DISPATCH.REGIONSUM — regional total demand and dispatchable generation", + entity_scope: "GridRegion (aemo_region_id)", + }, + { + feed_id: FEED_PRICE, + source_id: SOURCE_ID, + connector_id: CONNECTOR_ID, + metrics: ["market.price.energy"], + cadence: "5min", + description: "DISPATCH.PRICE — regional dispatch energy price (RRP)", + entity_scope: "GridRegion (aemo_region_id)", + }, + { + feed_id: FEED_INTERCONNECTOR, + source_id: SOURCE_ID, + connector_id: CONNECTOR_ID, + metrics: ["intercon.flow.mw"], + cadence: "5min", + description: "DISPATCH.INTERCONNECTORRES — signed interconnector MW flow", + entity_scope: "Interconnector (MMS INTERCONNECTORID kept in external_ids.aemo_duid)", + }, +]; + +function normalize(records: ParsedRecord[], ctx: ConnectorCtx): NormalizedOutput { + const observations: NormalizedOutput["observations"] = []; + const unmapped: string[] = []; + const ingestTime = ctx.now(); + const corrections = new CorrectionTracker(); + + for (const { table, row } of records) { + if (table !== "DISPATCH.REGIONSUM" && table !== "DISPATCH.PRICE" && table !== "DISPATCH.INTERCONNECTORRES") { + continue; // other DISPATCH tables (CASE_SOLUTION, CONSTRAINT, ...) are out of scope + } + const settlement = fieldStr(row, "SETTLEMENTDATE"); + const runno = fieldNum(row, "RUNNO") ?? 0; + const intervention = fieldNum(row, "INTERVENTION") ?? 0; + const eventTime = nemTimeToIso(settlement); + const sourceSequence = `${settlement}:${runno}`; + + if (table === "DISPATCH.INTERCONNECTORRES") { + const interconnectorId = fieldStr(row, "INTERCONNECTORID"); + const entityId = ctx.entityIndex.byDuid(interconnectorId); + if (entityId === undefined) { + reportUnmapped(unmapped, interconnectorId); + continue; + } + const flow = fieldNum(row, "MWFLOW"); + if (flow === null) continue; + observations.push( + makeObservation({ + entity_id: entityId, + metric: "intercon.flow.mw", + value: flow, // signed; positive = the interconnector's forward direction + unit: "MW", + event_time: eventTime, + ingest_time: ingestTime, + source_id: SOURCE_ID, + feed_id: FEED_INTERCONNECTOR, + source_sequence: sourceSequence, + is_correction: corrections.isCorrection(`${table}|${interconnectorId}|${settlement}`, runno, intervention), + meta: { interconnectorid: interconnectorId, intervention }, + }), + ); + continue; + } + + const regionId = fieldStr(row, "REGIONID"); + const entityId = ctx.entityIndex.byRegionId(regionId); + if (entityId === undefined) { + reportUnmapped(unmapped, regionId); + continue; + } + const isCorrection = corrections.isCorrection(`${table}|${regionId}|${settlement}`, runno, intervention); + const common = { + entity_id: entityId, + event_time: eventTime, + ingest_time: ingestTime, + source_id: SOURCE_ID, + source_sequence: sourceSequence, + is_correction: isCorrection, + meta: { regionid: regionId, intervention }, + }; + + if (table === "DISPATCH.PRICE") { + const rrp = fieldNum(row, "RRP"); + if (rrp !== null) { + observations.push( + makeObservation({ ...common, metric: "market.price.energy", value: rrp, unit: "AUD/MWh", feed_id: FEED_PRICE }), + ); + } + } else { + // DISPATCH.REGIONSUM + const demand = fieldNum(row, "TOTALDEMAND"); + if (demand !== null) { + observations.push( + makeObservation({ ...common, metric: "grid.demand.mw", value: demand, unit: "MW", feed_id: FEED_REGIONSUM }), + ); + } + // Prefer DISPATCHABLEGENERATION when the report carries it; fall back to + // AVAILABLEGENERATION otherwise. + const generation = fieldNum(row, "DISPATCHABLEGENERATION") ?? fieldNum(row, "AVAILABLEGENERATION"); + if (generation !== null) { + observations.push( + makeObservation({ ...common, metric: "grid.generation.mw", value: generation, unit: "MW", feed_id: FEED_REGIONSUM }), + ); + } + } + } + return { observations, unmapped }; +} + +export const aemoNemwebDispatchis: Connector = { + id: CONNECTOR_ID, + source_id: SOURCE_ID, + feeds, + schedule: { intervalMs: 5 * 60_000, jitterMs: 30_000 }, + discover: makeNemwebDiscover(BASE_URL, FILENAME_RE), + fetch: nemwebFetch, + parse: nemwebParseZip, + normalize, +}; diff --git a/intelligence/packages/connectors/src/connectors/aemo-nemweb-tradingis.ts b/intelligence/packages/connectors/src/connectors/aemo-nemweb-tradingis.ts new file mode 100644 index 0000000..41477b8 --- /dev/null +++ b/intelligence/packages/connectors/src/connectors/aemo-nemweb-tradingis.ts @@ -0,0 +1,79 @@ +// aemo-nemweb-tradingis — NEMWEB Current TradingIS_Reports (trading-interval +// price report). Wire format bound from the published MMS Data Model: table +// TRADING.PRICE (SETTLEMENTDATE, RUNNO, REGIONID, PERIODID, RRP). The feed is +// deliberately distinct from the dispatch price feed — trading price is a +// different market quantity than the 5-minute dispatch RRP. +// Built against the documented format only — fixtures are synthetic-from-spec. + +import type { TelemetryFeedRecord } from "@pact-tailor/ontology"; +import type { Connector, ConnectorCtx, NormalizedOutput, ParsedRecord } from "../sdk.js"; +import { nemTimeToIso } from "../aemo-csv.js"; +import { fieldNum, fieldStr, makeObservation, reportUnmapped } from "../observations.js"; +import { CorrectionTracker, makeNemwebDiscover, nemwebFetch, nemwebParseZip } from "./nemweb-common.js"; + +const CONNECTOR_ID = "aemo-nemweb-tradingis"; +const SOURCE_ID = "source:aemo:nemweb-tradingis"; +const BASE_URL = "https://nemweb.com.au/Reports/Current/TradingIS_Reports/"; +const FILENAME_RE = /PUBLIC_TRADINGIS_\d{12}_\d+\.zip/; + +const FEED_TRADING_PRICE = "feed:au-nem:trading:price"; + +const feeds: TelemetryFeedRecord[] = [ + { + feed_id: FEED_TRADING_PRICE, + source_id: SOURCE_ID, + connector_id: CONNECTOR_ID, + metrics: ["market.price.energy"], + cadence: "5min", + description: "TRADING.PRICE — regional trading-interval energy price (RRP)", + entity_scope: "GridRegion (aemo_region_id)", + }, +]; + +function normalize(records: ParsedRecord[], ctx: ConnectorCtx): NormalizedOutput { + const observations: NormalizedOutput["observations"] = []; + const unmapped: string[] = []; + const ingestTime = ctx.now(); + const corrections = new CorrectionTracker(); + + for (const { table, row } of records) { + if (table !== "TRADING.PRICE") continue; + const regionId = fieldStr(row, "REGIONID"); + const entityId = ctx.entityIndex.byRegionId(regionId); + if (entityId === undefined) { + reportUnmapped(unmapped, regionId); + continue; + } + const rrp = fieldNum(row, "RRP"); + if (rrp === null) continue; + const settlement = fieldStr(row, "SETTLEMENTDATE"); + const runno = fieldNum(row, "RUNNO") ?? 0; + observations.push( + makeObservation({ + entity_id: entityId, + metric: "market.price.energy", + value: rrp, + unit: "AUD/MWh", + event_time: nemTimeToIso(settlement), + ingest_time: ingestTime, + source_id: SOURCE_ID, + feed_id: FEED_TRADING_PRICE, + source_sequence: `${settlement}:${runno}`, + is_correction: corrections.isCorrection(`${regionId}|${settlement}`, runno, fieldNum(row, "INVALIDFLAG") ?? 0), + meta: { regionid: regionId, periodid: fieldStr(row, "PERIODID") }, + }), + ); + } + return { observations, unmapped }; +} + +export const aemoNemwebTradingis: Connector = { + id: CONNECTOR_ID, + source_id: SOURCE_ID, + feeds, + schedule: { intervalMs: 5 * 60_000, jitterMs: 30_000 }, + discover: makeNemwebDiscover(BASE_URL, FILENAME_RE), + fetch: nemwebFetch, + parse: nemwebParseZip, + normalize, +}; diff --git a/intelligence/packages/connectors/src/connectors/aemo-vis-nem-summary.ts b/intelligence/packages/connectors/src/connectors/aemo-vis-nem-summary.ts new file mode 100644 index 0000000..e32e4a1 --- /dev/null +++ b/intelligence/packages/connectors/src/connectors/aemo-vis-nem-summary.ts @@ -0,0 +1,149 @@ +// aemo-vis-nem-summary — the JSON backing service of AEMO's public NEM data +// dashboard. +// +// DOCUMENTED-FROM-PUBLIC-DASHBOARD: this endpoint has no formal API spec; the +// shape below is what the dashboard's own network calls exchange, as +// documented in the source registry record (source:aemo:vis-nem-summary). +// Treat NEMWEB DispatchIS as canonical; this is a convenience snapshot. +// +// GET https://visualisations.aemo.com.au/aemo/apps/api/report/ELEC_NEM_SUMMARY +// → { +// "ELEC_NEM_SUMMARY": [ +// { "SETTLEMENTDATE": "2026-08-12T11:05:00", // NEM time (UTC+10), no zone suffix +// "REGIONID": "QLD1", +// "TOTALDEMAND": 6123.45, // MW +// "SCHEDULEDGENERATION": 5410.2, // MW +// "SEMISCHEDULEDGENERATION": 489.9, // MW +// "PRICE": 84.5, // AUD/MWh +// ... }, +// ... one row per region ], +// "ELEC_NEM_SUMMARY_PRICES": [...], // sibling blocks not consumed here +// "ELEC_NEM_SUMMARY_MARKET_NOTICE": [...] +// } +// +// grid.generation.mw = SCHEDULEDGENERATION + SEMISCHEDULEDGENERATION. +// Built against the documented shape only — fixtures are synthetic-from-spec. + +import type { TelemetryFeedRecord } from "@pact-tailor/ontology"; +import type { ArtifactRef, Connector, ConnectorCtx, NormalizedOutput, ParsedRecord, RawArtifact } from "../sdk.js"; +import { nemTimeToIso } from "../aemo-csv.js"; +import { fieldNum, fieldStr, makeObservation, reportUnmapped } from "../observations.js"; + +const CONNECTOR_ID = "aemo-vis-nem-summary"; +const SOURCE_ID = "source:aemo:vis-nem-summary"; +const REPORT_URL = "https://visualisations.aemo.com.au/aemo/apps/api/report/ELEC_NEM_SUMMARY"; + +const FEED_SUMMARY = "feed:au-nem:vis-nem-summary"; + +const feeds: TelemetryFeedRecord[] = [ + { + feed_id: FEED_SUMMARY, + source_id: SOURCE_ID, + connector_id: CONNECTOR_ID, + metrics: ["grid.demand.mw", "grid.generation.mw", "market.price.energy"], + cadence: "5min", + description: "ELEC_NEM_SUMMARY dashboard snapshot — per-region demand, generation, price", + entity_scope: "GridRegion (aemo_region_id)", + notes: "Undocumented dashboard endpoint; NEMWEB DispatchIS is canonical for the same quantities.", + }, +]; + +async function discover(_ctx: ConnectorCtx): Promise { + return [{ url: REPORT_URL, name: "ELEC_NEM_SUMMARY" }]; +} + +async function fetchArtifact(ref: ArtifactRef, ctx: ConnectorCtx): Promise { + const res = await ctx.gate.get(ref.url); + if (res.status !== 200) { + throw new Error(`vis-nem-summary: GET ${ref.url} returned HTTP ${res.status}`); + } + return { + ref, + body: res.body, + fetched_at: ctx.now(), + http_status: res.status, + ...(res.content_type !== undefined ? { content_type: res.content_type } : {}), + }; +} + +async function parse(raw: RawArtifact): Promise { + const doc = JSON.parse(raw.body.toString("utf8")) as { ELEC_NEM_SUMMARY?: unknown }; + const rows = Array.isArray(doc.ELEC_NEM_SUMMARY) ? doc.ELEC_NEM_SUMMARY : []; + const records: ParsedRecord[] = []; + for (const raw of rows) { + if (raw === null || typeof raw !== "object") continue; + const src = raw as Record; + const row: Record = {}; + for (const key of [ + "SETTLEMENTDATE", + "REGIONID", + "TOTALDEMAND", + "SCHEDULEDGENERATION", + "SEMISCHEDULEDGENERATION", + "PRICE", + ]) { + const v = src[key]; + if (typeof v === "string" || typeof v === "number") row[key] = v; + } + records.push({ table: "ELEC_NEM_SUMMARY", row }); + } + return records; +} + +function normalize(records: ParsedRecord[], ctx: ConnectorCtx): NormalizedOutput { + const observations: NormalizedOutput["observations"] = []; + const unmapped: string[] = []; + const ingestTime = ctx.now(); + + for (const { table, row } of records) { + if (table !== "ELEC_NEM_SUMMARY") continue; + const regionId = fieldStr(row, "REGIONID"); + const entityId = ctx.entityIndex.byRegionId(regionId); + if (entityId === undefined) { + reportUnmapped(unmapped, regionId); + continue; + } + const settlement = fieldStr(row, "SETTLEMENTDATE"); + const common = { + entity_id: entityId, + event_time: nemTimeToIso(settlement), + ingest_time: ingestTime, + source_id: SOURCE_ID, + feed_id: FEED_SUMMARY, + source_sequence: settlement, + meta: { regionid: regionId }, + }; + const demand = fieldNum(row, "TOTALDEMAND"); + if (demand !== null) { + observations.push(makeObservation({ ...common, metric: "grid.demand.mw", value: demand, unit: "MW" })); + } + const scheduled = fieldNum(row, "SCHEDULEDGENERATION"); + const semiScheduled = fieldNum(row, "SEMISCHEDULEDGENERATION"); + if (scheduled !== null || semiScheduled !== null) { + observations.push( + makeObservation({ + ...common, + metric: "grid.generation.mw", + value: (scheduled ?? 0) + (semiScheduled ?? 0), + unit: "MW", + }), + ); + } + const price = fieldNum(row, "PRICE"); + if (price !== null) { + observations.push(makeObservation({ ...common, metric: "market.price.energy", value: price, unit: "AUD/MWh" })); + } + } + return { observations, unmapped }; +} + +export const aemoVisNemSummary: Connector = { + id: CONNECTOR_ID, + source_id: SOURCE_ID, + feeds, + schedule: { intervalMs: 5 * 60_000, jitterMs: 30_000 }, + discover, + fetch: fetchArtifact, + parse, + normalize, +}; diff --git a/intelligence/packages/connectors/src/connectors/aisstream.ts b/intelligence/packages/connectors/src/connectors/aisstream.ts new file mode 100644 index 0000000..879909f --- /dev/null +++ b/intelligence/packages/connectors/src/connectors/aisstream.ts @@ -0,0 +1,250 @@ +// aisstream — real-time AIS via aisstream.io's websocket, scoped to the +// Gladstone harbour approaches. +// +// Wire format bound from the published documentation +// (https://aisstream.io/documentation): +// +// subscribe (first message after connect): +// { "APIKey": "...", "BoundingBoxes": [[[lat1, lon1], [lat2, lon2]]], +// "FilterMessageTypes": ["PositionReport", "ShipStaticData"] } +// +// incoming message: +// { "MessageType": "PositionReport" | "ShipStaticData" | ..., +// "MetaData": { "MMSI": 503000001, "ShipName": "...", "latitude": ..., +// "longitude": ..., "time_utc": "2026-08-12 01:00:00.000000000 +0000 UTC" }, +// "Message": { "PositionReport": { "Sog": 8.2, "Cog": 123.4, +// "TrueHeading": 121, ... }, +// "ShipStaticData": { "MaximumStaticDraught": 10.5, ... } } } +// +// A fetch() collects one bounded window of messages through the MessageGate +// (replay: the committed JSONL fixture, zero delay; live: a real ws window) +// and stores them as one NDJSON bronze artifact. Built against the documented +// format only — fixtures are synthetic-from-spec, and the fixture MMSIs +// (503000001..503000003) are synthetic exemplar vessels, not real ships. + +import type { TelemetryFeedRecord, VesselPositionValue } from "@pact-tailor/ontology"; +import type { ArtifactRef, Connector, ConnectorCtx, NormalizedOutput, ParsedRecord, RawArtifact, VerificationReport } from "../sdk.js"; +import { fieldNum, fieldStr, makeObservation, reportUnmapped } from "../observations.js"; + +const CONNECTOR_ID = "aisstream"; +const SOURCE_ID = "source:aisstream:websocket"; +const STREAM_URL = "wss://stream.aisstream.io/v0/stream"; + +/** + * Gladstone harbour + approaches bounding box, [[lat1, lon1], [lat2, lon2]] + * (aisstream's documented coordinate order is [lat, lon]). + */ +export const GLADSTONE_BBOX: [[number, number], [number, number]] = [ + [-24.1, 150.9], + [-23.6, 151.6], +]; + +const FEED_POSITIONS = "feed:maritime:aisstream:gladstone"; + +const feeds: TelemetryFeedRecord[] = [ + { + feed_id: FEED_POSITIONS, + source_id: SOURCE_ID, + connector_id: CONNECTOR_ID, + metrics: ["vessel.position", "vessel.speed.kn", "vessel.draught.m"], + cadence: "streaming (per received AIS message)", + description: "AIS PositionReport/ShipStaticData for the Gladstone harbour bounding box", + entity_scope: "Vessel (external_ids.mmsi)", + }, +]; + +/** "2026-08-12 01:00:00.000000000 +0000 UTC" (Go time formatting) → ISO UTC. */ +export function aisTimeToIso(timeUtc: string): string { + const m = /^(\d{4})-(\d{2})-(\d{2})[ T](\d{2}):(\d{2}):(\d{2})(?:\.\d+)?\s+\+0000\s+UTC$/.exec(timeUtc.trim()); + if (!m) throw new Error(`aisstream: unrecognised time_utc "${timeUtc}"`); + const [, y, mo, d, h, mi, s] = m; + return `${y}-${mo}-${d}T${h}:${mi}:${s}Z`; +} + +async function discover(_ctx: ConnectorCtx): Promise { + return [{ url: STREAM_URL, name: "aisstream-gladstone-window", meta: { bbox: GLADSTONE_BBOX } }]; +} + +function subscribePayload(): Record { + return { + APIKey: process.env["AISSTREAM_API_KEY"] ?? "", + BoundingBoxes: [GLADSTONE_BBOX], + FilterMessageTypes: ["PositionReport", "ShipStaticData"], + }; +} + +async function fetchArtifact(ref: ArtifactRef, ctx: ConnectorCtx): Promise { + const gate = ctx.messageGate; + if (gate === undefined) { + throw new Error("aisstream: ConnectorCtx.messageGate is required (websocket source)"); + } + const messages = await gate.collect(ref.url, subscribePayload(), { windowMs: 30_000, maxMessages: 500 }); + return { + ref, + body: Buffer.from(messages.map((m) => m.toString("utf8")).join("\n"), "utf8"), + fetched_at: ctx.now(), + // Streams have no HTTP status; 200 records "window collected successfully". + http_status: 200, + content_type: "application/x-ndjson", + }; +} + +interface AisMessage { + MessageType?: string; + MetaData?: { + MMSI?: number; + ShipName?: string; + latitude?: number; + longitude?: number; + time_utc?: string; + }; + Message?: { + PositionReport?: { Sog?: number; Cog?: number; TrueHeading?: number }; + ShipStaticData?: { MaximumStaticDraught?: number; Destination?: string }; + }; +} + +async function parse(raw: RawArtifact): Promise { + const records: ParsedRecord[] = []; + for (const line of raw.body.toString("utf8").split("\n")) { + if (line.trim().length === 0) continue; + const msg = JSON.parse(line) as AisMessage; + const meta = msg.MetaData ?? {}; + const row: Record = { + MMSI: meta.MMSI ?? 0, + ShipName: (meta.ShipName ?? "").trim(), + latitude: meta.latitude ?? Number.NaN, + longitude: meta.longitude ?? Number.NaN, + time_utc: meta.time_utc ?? "", + }; + const position = msg.Message?.PositionReport; + if (position !== undefined) { + if (position.Sog !== undefined) row["Sog"] = position.Sog; + if (position.Cog !== undefined) row["Cog"] = position.Cog; + if (position.TrueHeading !== undefined) row["TrueHeading"] = position.TrueHeading; + } + const staticData = msg.Message?.ShipStaticData; + if (staticData !== undefined) { + if (staticData.MaximumStaticDraught !== undefined) row["MaximumStaticDraught"] = staticData.MaximumStaticDraught; + if (staticData.Destination !== undefined) row["Destination"] = staticData.Destination.trim(); + } + records.push({ table: msg.MessageType ?? "Unknown", row }); + } + return records; +} + +/** AIS "heading unavailable" sentinel per ITU-R M.1371. */ +const HEADING_UNAVAILABLE = 511; + +function normalize(records: ParsedRecord[], ctx: ConnectorCtx): NormalizedOutput { + const observations: NormalizedOutput["observations"] = []; + const unmapped: string[] = []; + const ingestTime = ctx.now(); + + for (const { table, row } of records) { + const mmsi = fieldStr(row, "MMSI"); + const entityId = ctx.entityIndex.byMmsi(mmsi); + if (entityId === undefined) { + reportUnmapped(unmapped, mmsi); + continue; + } + const timeUtc = fieldStr(row, "time_utc"); + const eventTime = aisTimeToIso(timeUtc); + const common = { + entity_id: entityId, + event_time: eventTime, + ingest_time: ingestTime, + source_id: SOURCE_ID, + feed_id: FEED_POSITIONS, + source_sequence: `${mmsi}:${timeUtc}`, + }; + + if (table === "PositionReport") { + const lat = fieldNum(row, "latitude"); + const lon = fieldNum(row, "longitude"); + if (lat === null || lon === null) continue; + const sog = fieldNum(row, "Sog"); + const heading = fieldNum(row, "TrueHeading"); + const value: VesselPositionValue = { + lat, + lon, + ...(sog !== null ? { speed_kn: sog } : {}), + ...(heading !== null && heading !== HEADING_UNAVAILABLE ? { heading_deg: heading } : {}), + }; + const cog = fieldNum(row, "Cog"); + observations.push( + makeObservation({ + ...common, + metric: "vessel.position", + value, + ...(cog !== null ? { meta: { cog } } : {}), + }), + ); + if (sog !== null) { + observations.push(makeObservation({ ...common, metric: "vessel.speed.kn", value: sog, unit: "kn" })); + } + } else if (table === "ShipStaticData") { + const draught = fieldNum(row, "MaximumStaticDraught"); + if (draught !== null && draught > 0) { + observations.push( + makeObservation({ + ...common, + metric: "vessel.draught.m", + value: draught, + unit: "m", + meta: { destination: fieldStr(row, "Destination") }, + }), + ); + } + } + } + return { observations, unmapped }; +} + +async function verify(ctx: ConnectorCtx): Promise { + const checkedAt = ctx.now(); + if ((process.env["AISSTREAM_API_KEY"] ?? "") === "") { + return { + connector_id: CONNECTOR_ID, + source_id: SOURCE_ID, + ok: false, + error: "AISSTREAM_API_KEY not set — aisstream requires a (free) registered API key", + checked_at: checkedAt, + }; + } + const gate = ctx.messageGate; + if (gate === undefined) { + return { connector_id: CONNECTOR_ID, source_id: SOURCE_ID, ok: false, error: "no MessageGate", checked_at: checkedAt }; + } + try { + const messages = await gate.collect(STREAM_URL, subscribePayload(), { windowMs: 10_000, maxMessages: 5 }); + return { + connector_id: CONNECTOR_ID, + source_id: SOURCE_ID, + ok: messages.length > 0, + ...(messages.length === 0 ? { error: "connected but received no messages in the probe window" } : {}), + checked_at: checkedAt, + notes: `${messages.length} message(s) in probe window`, + }; + } catch (err) { + return { + connector_id: CONNECTOR_ID, + source_id: SOURCE_ID, + ok: false, + error: (err as Error).message, + checked_at: checkedAt, + }; + } +} + +export const aisstream: Connector = { + id: CONNECTOR_ID, + source_id: SOURCE_ID, + feeds, + schedule: { intervalMs: 60_000, jitterMs: 10_000 }, + discover, + fetch: fetchArtifact, + parse, + normalize, + verify, +}; diff --git a/intelligence/packages/connectors/src/connectors/bom-observations.ts b/intelligence/packages/connectors/src/connectors/bom-observations.ts new file mode 100644 index 0000000..9d4d50e --- /dev/null +++ b/intelligence/packages/connectors/src/connectors/bom-observations.ts @@ -0,0 +1,181 @@ +// bom-observations — Bureau of Meteorology station observations (JSON data +// feed products). +// +// Wire format bound from the published product format (e.g. IDQ60801, +// Queensland observations; per-station JSON at +// www.bom.gov.au/fwo/{product}/{product}.{wmo-id}.json): +// +// { "observations": { +// "notice": [...], "header": [...], +// "data": [ { "wmo": 94381, "name": "Gladstone Airport", +// "aifstime_utc": "20260812013000", // UTC, yyyymmddHHMMSS +// "air_temp": 18.4, // degC +// "wind_spd_kmh": 19, ... }, ... ] } } +// +// Weather stations are connector-owned entities: the NEM structural seed does +// not include them, so this connector exports `seedEntities` (the Gladstone +// AWS Sensor record) for the CLI to merge into the graph load. Consistent +// with the no-guessing rule, normalize() still DROPS rows whose station is +// not in the EntityIndex and reports the wmo id in `unmapped` — the seed +// mechanism, not a hardcoded bypass, is what makes the station resolvable. +// Built against the documented format only — fixtures are synthetic-from-spec. + +import type { EntityRecord, TelemetryFeedRecord } from "@pact-tailor/ontology"; +import type { ArtifactRef, Connector, ConnectorCtx, NormalizedOutput, ParsedRecord, RawArtifact } from "../sdk.js"; +import { fieldNum, fieldStr, makeObservation, reportUnmapped } from "../observations.js"; + +const CONNECTOR_ID = "bom-observations"; +const SOURCE_ID = "source:bom:observations"; + +const FEED_QLD_OBS = "feed:weather:bom:qld-obs"; + +/** Hardcoded station map: BOM product/WMO id → connector-owned Sensor entity. */ +const STATIONS = [ + { + product: "IDQ60801", + wmo: "94381", + url: "http://www.bom.gov.au/fwo/IDQ60801/IDQ60801.94381.json", + entity_id: "sensor:au-qld:gladstone-aws", + name: "Gladstone Airport AWS", + lat: -23.87, + lon: 151.22, + }, +] as const; + +const feeds: TelemetryFeedRecord[] = [ + { + feed_id: FEED_QLD_OBS, + source_id: SOURCE_ID, + connector_id: CONNECTOR_ID, + metrics: ["weather.temp.c", "weather.wind.speed_ms"], + cadence: "30min (10min at some stations)", + description: "BOM AWS observations for the Gladstone industrial cluster", + entity_scope: "Sensor (connector-owned weather stations)", + }, +]; + +const seedEntities: EntityRecord[] = STATIONS.map((station) => ({ + entity_id: station.entity_id, + entity_type: "Sensor", + name: station.name, + data_class: "structural", + observability: "KNOWN_LIVE", + geometry: { type: "Point", coordinates: [station.lon, station.lat] }, + country: "au", + admin: { region: "qld", locality: "gladstone" }, + grid_id: null, + properties: { sensor_kind: "weather_station", bom_product: station.product, bom_wmo: station.wmo }, + external_ids: {}, + sources: [SOURCE_ID], + notes: + "Connector-owned weather-station entity seeded by the bom-observations connector (not part of the NEM structural seed).", +})); + +/** "20260812013000" (BOM aifstime_utc, already UTC) → ISO-8601 UTC. */ +export function bomTimeToIso(aifstimeUtc: string): string { + const m = /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})$/.exec(aifstimeUtc.trim()); + if (!m) throw new Error(`bom-observations: unrecognised aifstime_utc "${aifstimeUtc}"`); + const [, y, mo, d, h, mi, s] = m; + return `${y}-${mo}-${d}T${h}:${mi}:${s}Z`; +} + +async function discover(_ctx: ConnectorCtx): Promise { + return STATIONS.map((station) => ({ + url: station.url, + name: `${station.product}.${station.wmo}`, + meta: { wmo: station.wmo }, + })); +} + +async function fetchArtifact(ref: ArtifactRef, ctx: ConnectorCtx): Promise { + const res = await ctx.gate.get(ref.url); + if (res.status !== 200) { + throw new Error(`bom-observations: GET ${ref.url} returned HTTP ${res.status}`); + } + return { + ref, + body: res.body, + fetched_at: ctx.now(), + http_status: res.status, + ...(res.content_type !== undefined ? { content_type: res.content_type } : {}), + }; +} + +async function parse(raw: RawArtifact): Promise { + const doc = JSON.parse(raw.body.toString("utf8")) as { observations?: { data?: unknown } }; + const data = doc.observations?.data; + const rows = Array.isArray(data) ? data : []; + const records: ParsedRecord[] = []; + for (const entry of rows) { + if (entry === null || typeof entry !== "object") continue; + const src = entry as Record; + const row: Record = {}; + for (const key of ["wmo", "name", "aifstime_utc", "air_temp", "wind_spd_kmh"]) { + const v = src[key]; + // BOM uses null / "-" for missing values; keep only real strings/numbers. + if (typeof v === "number" || (typeof v === "string" && v !== "-")) row[key] = v; + } + records.push({ table: "observations", row }); + } + return records; +} + +const KMH_TO_MS = 1 / 3.6; + +function normalize(records: ParsedRecord[], ctx: ConnectorCtx): NormalizedOutput { + const observations: NormalizedOutput["observations"] = []; + const unmapped: string[] = []; + const ingestTime = ctx.now(); + const stationByWmo = new Map(STATIONS.map((s) => [s.wmo, s])); + + for (const { table, row } of records) { + if (table !== "observations") continue; + const wmo = fieldStr(row, "wmo"); + const station = stationByWmo.get(wmo); + // Unknown station, or station entity absent from the index (seed not + // merged): DROP and report — never emit against an unresolvable entity. + if (station === undefined || !ctx.entityIndex.has(station.entity_id)) { + reportUnmapped(unmapped, wmo); + continue; + } + const aifstime = fieldStr(row, "aifstime_utc"); + if (aifstime === "") continue; + const common = { + entity_id: station.entity_id, + event_time: bomTimeToIso(aifstime), + ingest_time: ingestTime, + source_id: SOURCE_ID, + feed_id: FEED_QLD_OBS, + source_sequence: `${wmo}:${aifstime}`, + meta: { wmo, station: fieldStr(row, "name") }, + }; + const airTemp = fieldNum(row, "air_temp"); + if (airTemp !== null) { + observations.push(makeObservation({ ...common, metric: "weather.temp.c", value: airTemp, unit: "degC" })); + } + const windKmh = fieldNum(row, "wind_spd_kmh"); + if (windKmh !== null) { + observations.push( + makeObservation({ + ...common, + metric: "weather.wind.speed_ms", + value: Math.round(windKmh * KMH_TO_MS * 100) / 100, + unit: "m/s", + }), + ); + } + } + return { observations, unmapped }; +} + +export const bomObservations: Connector = { + id: CONNECTOR_ID, + source_id: SOURCE_ID, + feeds, + schedule: { intervalMs: 30 * 60_000, jitterMs: 120_000 }, + discover, + fetch: fetchArtifact, + parse, + normalize, + seedEntities, +}; diff --git a/intelligence/packages/connectors/src/connectors/eia-v2.ts b/intelligence/packages/connectors/src/connectors/eia-v2.ts new file mode 100644 index 0000000..004e71d --- /dev/null +++ b/intelligence/packages/connectors/src/connectors/eia-v2.ts @@ -0,0 +1,148 @@ +// eia-v2 (STUB) — U.S. EIA API v2, electricity/rto/region-data (hourly demand +// by balancing authority). +// +// STUB POSTURE: EIA v2 requires an api_key (freely issued, but still a +// registration credential). Live discovery returns [] with a log line and a +// live fetch throws; parse/normalize are fully implemented against the +// DOCUMENTED response shape and exercised via the replay fixture +// (synthetic-from-spec). +// +// Documented response shape (EIA API v2 documentation): +// +// { "response": { +// "total": ..., +// "data": [ { "period": "2026-08-12T00", // hourly, UTC +// "respondent": "DEMO", // balancing-authority code +// "respondent-name": "...", +// "type": "D", "type-name": "Demand", +// "value": 21450, +// "value-units": "megawatthours" }, ... ] }, +// "request": {...}, "apiVersion": "2.x.x" } +// +// Hourly demand in MWh over one hour is numerically the average MW for that +// hour; emitted as grid.demand.mw with the conversion noted in meta. + +import type { TelemetryFeedRecord } from "@pact-tailor/ontology"; +import type { ArtifactRef, Connector, ConnectorCtx, NormalizedOutput, ParsedRecord, RawArtifact, VerificationReport } from "../sdk.js"; +import { fieldNum, fieldStr, makeObservation, reportUnmapped } from "../observations.js"; + +const CONNECTOR_ID = "eia-v2"; +const SOURCE_ID = "source:eia:api-v2"; +const API_URL = + "https://api.eia.gov/v2/electricity/rto/region-data/data/?frequency=hourly&data[0]=value&facets[type][]=D"; + +const FEED_DEMAND = "feed:us:eia:demand"; + +const feeds: TelemetryFeedRecord[] = [ + { + feed_id: FEED_DEMAND, + source_id: SOURCE_ID, + connector_id: CONNECTOR_ID, + metrics: ["grid.demand.mw"], + cadence: "hourly", + description: "EIA-930 hourly demand per balancing authority via API v2", + entity_scope: "Grid/GridRegion balancing authorities (properties.eia_ba_code — see EntityIndex friction note)", + notes: "STUB — api_key-required source; live path disabled until EIA_API_KEY is configured.", + }, +]; + +async function discover(ctx: ConnectorCtx): Promise { + if (ctx.gate.kind === "live") { + ctx.log("eia-v2: STUB — live discovery requires an EIA api_key; returning no artifacts"); + return []; + } + return [{ url: API_URL, name: "region-data-demand" }]; +} + +async function fetchArtifact(ref: ArtifactRef, ctx: ConnectorCtx): Promise { + if (ctx.gate.kind === "live") { + throw new Error("eia-v2: live fetch requires an EIA api_key (stub connector)"); + } + const res = await ctx.gate.get(ref.url); + if (res.status !== 200) { + throw new Error(`eia-v2: GET ${ref.url} returned HTTP ${res.status}`); + } + return { + ref, + body: res.body, + fetched_at: ctx.now(), + http_status: res.status, + ...(res.content_type !== undefined ? { content_type: res.content_type } : {}), + }; +} + +async function parse(raw: RawArtifact): Promise { + const doc = JSON.parse(raw.body.toString("utf8")) as { response?: { data?: unknown } }; + const data = doc.response?.data; + const rows = Array.isArray(data) ? data : []; + const records: ParsedRecord[] = []; + for (const entry of rows) { + if (entry === null || typeof entry !== "object") continue; + const src = entry as Record; + const row: Record = {}; + for (const key of ["period", "respondent", "respondent-name", "type", "value", "value-units"]) { + const v = src[key]; + if (typeof v === "string" || typeof v === "number") row[key] = v; + } + records.push({ table: "region-data", row }); + } + return records; +} + +function normalize(records: ParsedRecord[], ctx: ConnectorCtx): NormalizedOutput { + const observations: NormalizedOutput["observations"] = []; + const unmapped: string[] = []; + const ingestTime = ctx.now(); + + for (const { table, row } of records) { + if (table !== "region-data") continue; + if (fieldStr(row, "type") !== "D") continue; // demand rows only + const respondent = fieldStr(row, "respondent"); + const entityId = ctx.entityIndex.byEiaBa(respondent); + if (entityId === undefined) { + reportUnmapped(unmapped, respondent); + continue; + } + const value = fieldNum(row, "value"); + if (value === null) continue; + const period = fieldStr(row, "period"); // "2026-08-12T00" (hourly) + if (!/^\d{4}-\d{2}-\d{2}T\d{2}$/.test(period)) continue; + observations.push( + makeObservation({ + entity_id: entityId, + metric: "grid.demand.mw", + value, + unit: "MW", + event_time: `${period}:00:00Z`, + ingest_time: ingestTime, + source_id: SOURCE_ID, + feed_id: FEED_DEMAND, + source_sequence: `${respondent}:${period}`, + meta: { respondent, source_units: fieldStr(row, "value-units"), conversion: "MWh/h == avg MW over the hour" }, + }), + ); + } + return { observations, unmapped }; +} + +async function verify(ctx: ConnectorCtx): Promise { + return { + connector_id: CONNECTOR_ID, + source_id: SOURCE_ID, + ok: false, + error: "STUB: requires EIA api_key (free registration at eia.gov/opendata) — live verification not attempted", + checked_at: ctx.now(), + }; +} + +export const eiaV2: Connector = { + id: CONNECTOR_ID, + source_id: SOURCE_ID, + feeds, + schedule: { intervalMs: 60 * 60_000, jitterMs: 300_000 }, + discover, + fetch: fetchArtifact, + parse, + normalize, + verify, +}; diff --git a/intelligence/packages/connectors/src/connectors/entsoe-transparency.ts b/intelligence/packages/connectors/src/connectors/entsoe-transparency.ts new file mode 100644 index 0000000..c0a3ce0 --- /dev/null +++ b/intelligence/packages/connectors/src/connectors/entsoe-transparency.ts @@ -0,0 +1,184 @@ +// entsoe-transparency (STUB) — ENTSO-E Transparency Platform, document type +// A75 (actual generation per production type). +// +// STUB POSTURE: the Transparency REST API is registration-required (free +// securityToken). Live discovery therefore returns [] with a log line, and a +// live fetch throws — no token, no fetch, per LEGAL-LICENSING.md. The parse +// and normalize paths are fully implemented against the DOCUMENTED XML format +// and exercised through the replay fixture (synthetic-from-spec). +// +// Documented A75 GL_MarketDocument shape (per the published Transparency user +// guide): +// +// +// +// 10Y... +// B05 +// +// 2026-08-12T00:00Z... +// PT60M +// 1640 +// ... +// +// +// +// +// The XML is consumed with a tiny hand-rolled tag scanner (no XML dependency) +// — sufficient for this well-formed, namespaced-but-flat document family. + +import type { TelemetryFeedRecord } from "@pact-tailor/ontology"; +import type { ArtifactRef, Connector, ConnectorCtx, NormalizedOutput, ParsedRecord, RawArtifact, VerificationReport } from "../sdk.js"; +import { fieldNum, fieldStr, makeObservation, reportUnmapped } from "../observations.js"; + +const CONNECTOR_ID = "entsoe-transparency"; +const SOURCE_ID = "source:entsoe:transparency-platform"; +const API_URL = "https://web-api.tp.entsoe.eu/api?documentType=A75&processType=A16"; + +const FEED_GENERATION = "feed:eu:entsoe:actual-generation"; + +const feeds: TelemetryFeedRecord[] = [ + { + feed_id: FEED_GENERATION, + source_id: SOURCE_ID, + connector_id: CONNECTOR_ID, + metrics: ["grid.generation.mw"], + cadence: "15-60min depending on bidding zone", + description: "A75 actual generation per production type, per bidding zone", + entity_scope: "Grid/GridRegion (external_ids.entsoe_eic)", + notes: "STUB — registration-required source; live path disabled until a securityToken is configured.", + }, +]; + +// --- tiny hand-rolled tag scanner (documented format, no xml dependency) --- + +function escapeTag(tag: string): string { + return tag.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** All inner contents of ..., non-nested, document order. */ +export function xmlBlocks(xml: string, tag: string): string[] { + const re = new RegExp(`<${escapeTag(tag)}(?:\\s[^>]*)?>([\\s\\S]*?)`, "g"); + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = re.exec(xml)) !== null) out.push(m[1] ?? ""); + return out; +} + +/** Trimmed text of the first ..., or undefined. */ +export function xmlText(xml: string, tag: string): string | undefined { + const first = xmlBlocks(xml, tag)[0]; + return first === undefined ? undefined : first.trim(); +} + +function resolutionMinutes(resolution: string): number { + const m = /^PT(\d+)M$/.exec(resolution); + if (!m) throw new Error(`entsoe-transparency: unsupported resolution "${resolution}"`); + return Number(m[1]); +} + +async function discover(ctx: ConnectorCtx): Promise { + if (ctx.gate.kind === "live") { + ctx.log("entsoe-transparency: STUB — live discovery requires a registered securityToken; returning no artifacts"); + return []; + } + return [{ url: API_URL, name: "a75-actual-generation" }]; +} + +async function fetchArtifact(ref: ArtifactRef, ctx: ConnectorCtx): Promise { + if (ctx.gate.kind === "live") { + throw new Error("entsoe-transparency: live fetch requires securityToken (registration-required source; stub connector)"); + } + const res = await ctx.gate.get(ref.url); + if (res.status !== 200) { + throw new Error(`entsoe-transparency: GET ${ref.url} returned HTTP ${res.status}`); + } + return { + ref, + body: res.body, + fetched_at: ctx.now(), + http_status: res.status, + ...(res.content_type !== undefined ? { content_type: res.content_type } : {}), + }; +} + +async function parse(raw: RawArtifact): Promise { + const xml = raw.body.toString("utf8"); + const records: ParsedRecord[] = []; + for (const series of xmlBlocks(xml, "TimeSeries")) { + const eic = xmlText(series, "inBiddingZone_Domain.mRID") ?? xmlText(series, "outBiddingZone_Domain.mRID") ?? ""; + const psrType = xmlText(series, "psrType") ?? ""; + for (const period of xmlBlocks(series, "Period")) { + const interval = xmlBlocks(period, "timeInterval")[0] ?? ""; + const start = xmlText(interval, "start") ?? ""; + const resolutionMin = resolutionMinutes(xmlText(period, "resolution") ?? ""); + for (const point of xmlBlocks(period, "Point")) { + const position = Number(xmlText(point, "position") ?? "0"); + const quantity = Number(xmlText(point, "quantity") ?? ""); + records.push({ + table: "A75", + row: { eic, psr_type: psrType, start, resolution_min: resolutionMin, position, quantity }, + }); + } + } + } + return records; +} + +function normalize(records: ParsedRecord[], ctx: ConnectorCtx): NormalizedOutput { + const observations: NormalizedOutput["observations"] = []; + const unmapped: string[] = []; + const ingestTime = ctx.now(); + + for (const { table, row } of records) { + if (table !== "A75") continue; + const eic = fieldStr(row, "eic"); + const entityId = ctx.entityIndex.byEic(eic); + if (entityId === undefined) { + reportUnmapped(unmapped, eic); + continue; + } + const quantity = fieldNum(row, "quantity"); + const position = fieldNum(row, "position"); + const resolutionMin = fieldNum(row, "resolution_min"); + const startMs = Date.parse(fieldStr(row, "start")); + if (quantity === null || position === null || resolutionMin === null || !Number.isFinite(startMs)) continue; + const eventMs = startMs + (position - 1) * resolutionMin * 60_000; + observations.push( + makeObservation({ + entity_id: entityId, + metric: "grid.generation.mw", + value: quantity, + unit: "MW", + event_time: new Date(eventMs).toISOString().replace(".000Z", "Z"), + ingest_time: ingestTime, + source_id: SOURCE_ID, + feed_id: FEED_GENERATION, + source_sequence: `${eic}:${fieldStr(row, "start")}:${position}`, + meta: { psr_type: fieldStr(row, "psr_type") }, + }), + ); + } + return { observations, unmapped }; +} + +async function verify(ctx: ConnectorCtx): Promise { + return { + connector_id: CONNECTOR_ID, + source_id: SOURCE_ID, + ok: false, + error: "STUB: requires securityToken (free registration at transparency.entsoe.eu) — live verification not attempted", + checked_at: ctx.now(), + }; +} + +export const entsoeTransparency: Connector = { + id: CONNECTOR_ID, + source_id: SOURCE_ID, + feeds, + schedule: { intervalMs: 15 * 60_000, jitterMs: 60_000 }, + discover, + fetch: fetchArtifact, + parse, + normalize, + verify, +}; diff --git a/intelligence/packages/connectors/src/connectors/index.ts b/intelligence/packages/connectors/src/connectors/index.ts new file mode 100644 index 0000000..0848200 --- /dev/null +++ b/intelligence/packages/connectors/src/connectors/index.ts @@ -0,0 +1,37 @@ +// Connector inventory. Order is the CLI's run order. + +import type { Connector } from "../sdk.js"; +import { aemoNemwebDispatchis } from "./aemo-nemweb-dispatchis.js"; +import { aemoNemwebDispatchScada } from "./aemo-nemweb-dispatch-scada.js"; +import { aemoNemwebTradingis } from "./aemo-nemweb-tradingis.js"; +import { aemoVisNemSummary } from "./aemo-vis-nem-summary.js"; +import { aisstream } from "./aisstream.js"; +import { bomObservations } from "./bom-observations.js"; +import { entsoeTransparency } from "./entsoe-transparency.js"; +import { eiaV2 } from "./eia-v2.js"; + +export { + aemoNemwebDispatchis, + aemoNemwebDispatchScada, + aemoNemwebTradingis, + aemoVisNemSummary, + aisstream, + bomObservations, + entsoeTransparency, + eiaV2, +}; +export { GLADSTONE_BBOX, aisTimeToIso } from "./aisstream.js"; +export { bomTimeToIso } from "./bom-observations.js"; +export { xmlBlocks, xmlText } from "./entsoe-transparency.js"; +export { CorrectionTracker, makeNemwebDiscover, nemwebFetch, nemwebParseZip } from "./nemweb-common.js"; + +export const allConnectors: Connector[] = [ + aemoNemwebDispatchis, + aemoNemwebDispatchScada, + aemoNemwebTradingis, + aemoVisNemSummary, + aisstream, + bomObservations, + entsoeTransparency, + eiaV2, +]; diff --git a/intelligence/packages/connectors/src/connectors/nemweb-common.ts b/intelligence/packages/connectors/src/connectors/nemweb-common.ts new file mode 100644 index 0000000..f5162e3 --- /dev/null +++ b/intelligence/packages/connectors/src/connectors/nemweb-common.ts @@ -0,0 +1,76 @@ +// Shared NEMWEB behaviour (documented file-drop layout at +// https://nemweb.com.au/Reports/Current/...): each report directory is an +// IIS-style HTML listing of ZIP files named +// PUBLIC___.zip, each ZIP containing a single +// AEMO MMS CSV report. The timestamp in the filename sorts lexicographically, +// so "newest" is the max filename. + +import type { ArtifactRef, ConnectorCtx, ParsedRecord, RawArtifact } from "../sdk.js"; +import { parseAemoCsv } from "../aemo-csv.js"; +import { unzipFirst } from "../zip.js"; + +/** Discover the newest report ZIP in a NEMWEB Current directory listing. */ +export function makeNemwebDiscover(baseUrl: string, filenameRe: RegExp): (ctx: ConnectorCtx) => Promise { + return async (ctx: ConnectorCtx): Promise => { + const res = await ctx.gate.get(baseUrl); + if (res.status !== 200) { + throw new Error(`nemweb: directory listing ${baseUrl} returned HTTP ${res.status}`); + } + const html = res.body.toString("utf8"); + const names = new Set(); + const re = new RegExp(filenameRe.source, "g"); + let m: RegExpExecArray | null; + while ((m = re.exec(html)) !== null) names.add(m[0]); + const sorted = [...names].sort(); + const newest = sorted[sorted.length - 1]; + if (newest === undefined) return []; + return [{ url: baseUrl + newest, name: newest }]; + }; +} + +/** Fetch a NEMWEB ZIP artifact verbatim. */ +export async function nemwebFetch(ref: ArtifactRef, ctx: ConnectorCtx): Promise { + const res = await ctx.gate.get(ref.url); + if (res.status !== 200) { + throw new Error(`nemweb: GET ${ref.url} returned HTTP ${res.status}`); + } + return { + ref, + body: res.body, + fetched_at: ctx.now(), + http_status: res.status, + ...(res.content_type !== undefined ? { content_type: res.content_type } : {}), + }; +} + +/** Unzip the single CSV report and flatten its MMS tables into ParsedRecords. */ +export async function nemwebParseZip(raw: RawArtifact): Promise { + const entry = unzipFirst(raw.body); + const tables = parseAemoCsv(entry.data.toString("utf8")); + const records: ParsedRecord[] = []; + for (const table of tables) { + for (const row of table.rows) records.push({ table: table.table, row }); + } + return records; +} + +/** + * Correction detection (documented simplification): AEMO re-runs a dispatch + * interval by publishing further rows for the same SETTLEMENTDATE with a + * higher RUNNO, and intervention runs carry INTERVENTION=1. WITHIN one batch + * we mark a row as a correction when its INTERVENTION flag is non-zero or a + * lower RUNNO for the same (table, entity, SETTLEMENTDATE) key was already + * seen. Cross-batch correction detection (against silver history) is + * deliberately out of scope here — the state engine reconciles batches by + * `source_sequence` (= "SETTLEMENTDATE:RUNNO"). + */ +export class CorrectionTracker { + private readonly firstRunno = new Map(); + + isCorrection(key: string, runno: number, intervention: number): boolean { + const first = this.firstRunno.get(key); + if (first === undefined) this.firstRunno.set(key, runno); + if (intervention > 0) return true; + return first !== undefined && runno > first; + } +} diff --git a/intelligence/packages/connectors/src/entity-index.ts b/intelligence/packages/connectors/src/entity-index.ts new file mode 100644 index 0000000..e75f952 --- /dev/null +++ b/intelligence/packages/connectors/src/entity-index.ts @@ -0,0 +1,74 @@ +// EntityIndex — external-id → entity_id resolution. +// +// Built once from EntityRecord[] (registry + connector seedEntities). Lookups +// return the canonical entity_id or undefined; connectors DROP observations +// whose external id does not resolve and report the id in `unmapped`. No +// guessing, ever — an unresolved id is an honest gap, not a new entity. + +import type { EntityRecord } from "@pact-tailor/ontology"; + +export class EntityIndex { + private readonly duid = new Map(); + private readonly region = new Map(); + private readonly mmsi = new Map(); + private readonly imo = new Map(); + private readonly eic = new Map(); + private readonly eiaBa = new Map(); + private readonly ids = new Set(); + + constructor(entities: EntityRecord[]) { + for (const entity of entities) { + this.ids.add(entity.entity_id); + const ext = entity.external_ids ?? {}; + for (const duid of ext.aemo_duid ?? []) this.duid.set(duid, entity.entity_id); + if (ext.aemo_region_id) this.region.set(ext.aemo_region_id, entity.entity_id); + if (ext.mmsi) this.mmsi.set(ext.mmsi, entity.entity_id); + if (ext.imo) this.imo.set(ext.imo, entity.entity_id); + if (ext.entsoe_eic) this.eic.set(ext.entsoe_eic, entity.entity_id); + // FRICTION NOTE: ontology's ExternalIds has no slot for an EIA balancing- + // authority code (eia_plant_code is a plant, not a BA). Rather than widen + // the shared type from this package, BA codes are read from the open + // properties bag (properties.eia_ba_code) — flagged for the ontology + // owner as a candidate ExternalIds field. + const eiaBa = entity.properties?.["eia_ba_code"]; + if (typeof eiaBa === "string" && eiaBa.length > 0) this.eiaBa.set(eiaBa, entity.entity_id); + } + } + + /** AEMO dispatchable unit id (also MMS INTERCONNECTORID, e.g. NSW1-QLD1). */ + byDuid(duid: string): string | undefined { + return this.duid.get(duid); + } + + /** AEMO region id, e.g. QLD1 → region:au-nem:qld1. */ + byRegionId(regionId: string): string | undefined { + return this.region.get(regionId); + } + + byMmsi(mmsi: string): string | undefined { + return this.mmsi.get(mmsi); + } + + byImo(imo: string): string | undefined { + return this.imo.get(imo); + } + + /** ENTSO-E EIC code (external_ids.entsoe_eic). */ + byEic(eic: string): string | undefined { + return this.eic.get(eic); + } + + /** EIA balancing-authority code (properties.eia_ba_code — see friction note). */ + byEiaBa(code: string): string | undefined { + return this.eiaBa.get(code); + } + + /** True if the entity_id itself is known (used by connector-owned entities). */ + has(entityId: string): boolean { + return this.ids.has(entityId); + } + + get size(): number { + return this.ids.size; + } +} diff --git a/intelligence/packages/connectors/src/gate.ts b/intelligence/packages/connectors/src/gate.ts new file mode 100644 index 0000000..454ce14 --- /dev/null +++ b/intelligence/packages/connectors/src/gate.ts @@ -0,0 +1,172 @@ +// HttpGate — the replay/live seam. Connectors call gate.get(url) and never +// know whether bytes came from the network or from committed fixtures; replay +// is the SAME code path as live. ReplayGate serves synthetic exchanges +// (fixture_provenance: synthetic-from-spec) and answers unmatched URLs with +// status 599 so a connector's gaps surface honestly instead of silently +// passing. + +import { existsSync, readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; + +export interface GateResponse { + status: number; + body: Buffer; + content_type?: string; +} + +export interface HttpGate { + get(url: string): Promise; + kind: "replay" | "live"; +} + +/** Synthetic status for "no fixture matched this URL" in replay mode. */ +export const REPLAY_UNMATCHED_STATUS = 599; + +/** Live HTTP: global fetch, 30s timeout, one retry. */ +export class LiveGate implements HttpGate { + readonly kind = "live" as const; + + constructor(private readonly timeoutMs = 30_000) {} + + async get(url: string): Promise { + let lastError: unknown; + for (let attempt = 0; attempt < 2; attempt++) { + try { + const res = await fetch(url, { + signal: AbortSignal.timeout(this.timeoutMs), + redirect: "follow", + headers: { "user-agent": "pact-tailor-intelligence/0.1 (connector ingest)" }, + }); + const body = Buffer.from(await res.arrayBuffer()); + const contentType = res.headers.get("content-type"); + return { + status: res.status, + body, + ...(contentType ? { content_type: contentType } : {}), + }; + } catch (err) { + lastError = err; + } + } + throw new Error(`LiveGate: GET ${url} failed after retry — ${(lastError as Error)?.message ?? String(lastError)}`); + } +} + +/** + * One committed exchange: fixtures/{connector_id}/exchanges/{NN}-{slug}.json. + * `url_pattern` is an exact URL or a URL prefix. + */ +export interface ReplayExchange { + url_pattern: string; + status: number; + content_type?: string; + body_base64: string; +} + +/** Serves committed synthetic exchanges. Exact URL match first, then longest prefix. */ +export class ReplayGate implements HttpGate { + readonly kind = "replay" as const; + private readonly exchanges: ReplayExchange[] = []; + + constructor(fixturesDir: string) { + const dir = join(fixturesDir, "exchanges"); + if (!existsSync(dir)) return; + for (const name of readdirSync(dir).sort()) { + if (!name.endsWith(".json")) continue; + this.exchanges.push(JSON.parse(readFileSync(join(dir, name), "utf8")) as ReplayExchange); + } + } + + async get(url: string): Promise { + const exact = this.exchanges.find((e) => e.url_pattern === url); + const match = + exact ?? + this.exchanges + .filter((e) => url.startsWith(e.url_pattern)) + .sort((a, b) => b.url_pattern.length - a.url_pattern.length)[0]; + if (!match) return { status: REPLAY_UNMATCHED_STATUS, body: Buffer.alloc(0) }; + return { + status: match.status, + body: Buffer.from(match.body_base64, "base64"), + ...(match.content_type ? { content_type: match.content_type } : {}), + }; + } +} + +// --------------------------------------------------------------------------- +// MessageGate — the same seam for streaming (websocket) sources. +// --------------------------------------------------------------------------- + +export interface MessageCollectOptions { + /** Stop after this many messages (live default 500). */ + maxMessages?: number; + /** Stop after this long (live default 30s). Replay delivers with zero delay. */ + windowMs?: number; +} + +export interface MessageGate { + kind: "replay" | "live"; + /** + * Open the stream, send the subscription payload, and collect raw messages + * until the stream ends (replay: fixture exhausted) or a limit is hit. + */ + collect(url: string, subscribe: unknown, opts?: MessageCollectOptions): Promise; +} + +/** + * Replays a committed JSONL fixture (fixtures/{connector_id}/messages.jsonl, + * one message per line) with zero delay. The subscription payload is accepted + * and ignored — fixtures already reflect the subscribed filter. + */ +export class ReplayMessageGate implements MessageGate { + readonly kind = "replay" as const; + + constructor(private readonly jsonlPath: string) {} + + async collect(_url: string, _subscribe: unknown, opts: MessageCollectOptions = {}): Promise { + if (!existsSync(this.jsonlPath)) return []; + const lines = readFileSync(this.jsonlPath, "utf8") + .split("\n") + .filter((line) => line.trim().length > 0); + const max = opts.maxMessages ?? lines.length; + return lines.slice(0, max).map((line) => Buffer.from(line, "utf8")); + } +} + +/** Live websocket collection via `ws` (loaded lazily so replay never needs it). */ +export class LiveMessageGate implements MessageGate { + readonly kind = "live" as const; + + async collect(url: string, subscribe: unknown, opts: MessageCollectOptions = {}): Promise { + const { WebSocket } = await import("ws"); + const maxMessages = opts.maxMessages ?? 500; + const windowMs = opts.windowMs ?? 30_000; + return new Promise((resolve, reject) => { + const messages: Buffer[] = []; + const socket = new WebSocket(url); + let settled = false; + const finish = (err?: Error) => { + if (settled) return; + settled = true; + clearTimeout(timer); + try { + socket.close(); + } catch { + // already closed + } + if (err && messages.length === 0) reject(err); + else resolve(messages); + }; + const timer = setTimeout(() => finish(), windowMs); + socket.on("open", () => { + socket.send(JSON.stringify(subscribe)); + }); + socket.on("message", (data) => { + messages.push(Buffer.isBuffer(data) ? data : Buffer.from(data as ArrayBuffer)); + if (messages.length >= maxMessages) finish(); + }); + socket.on("error", (err) => finish(err instanceof Error ? err : new Error(String(err)))); + socket.on("close", () => finish()); + }); + } +} diff --git a/intelligence/packages/connectors/src/index.ts b/intelligence/packages/connectors/src/index.ts index 83e8fa8..2543853 100644 --- a/intelligence/packages/connectors/src/index.ts +++ b/intelligence/packages/connectors/src/index.ts @@ -1,2 +1,13 @@ -// @pact-tailor/connectors — connector SDK and connectors. Populated in the connector commit. -export {}; +// @pact-tailor/connectors — connector SDK (discover/fetch/parse/normalize, +// replay|live HttpGate seam) plus the AEMO, AIS, BOM, ENTSO-E and EIA +// connectors. All fixtures in this package are synthetic-from-spec; nothing +// here claims live verification (see verify-live.ts). + +export * from "./sdk.js"; +export * from "./gate.js"; +export { EntityIndex } from "./entity-index.js"; +export { unzipFirst, type ZipEntry } from "./zip.js"; +export { parseAemoCsv, splitCsvLine, nemTimeToIso, type AemoTable } from "./aemo-csv.js"; +export { makeObservation, fieldStr, fieldNum, reportUnmapped, type ObsInput } from "./observations.js"; +export { runOnce, Scheduler, type IngestReport } from "./scheduler.js"; +export * from "./connectors/index.js"; diff --git a/intelligence/packages/connectors/src/observations.ts b/intelligence/packages/connectors/src/observations.ts new file mode 100644 index 0000000..bcffd2c --- /dev/null +++ b/intelligence/packages/connectors/src/observations.ts @@ -0,0 +1,58 @@ +// Small shared helpers for building Observation envelopes from parsed rows. + +import { ulid } from "@pact-tailor/ontology"; +import type { ObservationRecord, ObservationValue, Quality } from "@pact-tailor/ontology"; + +export interface ObsInput { + entity_id: string; + metric: string; + value: ObservationValue; + event_time: string; + ingest_time: string; + source_id: string; + feed_id: string; + unit?: string; + quality?: Quality; + is_correction?: boolean; + source_sequence?: string; + meta?: Record; +} + +/** Build a full ObservationRecord (observation_id is a ULID at event time). */ +export function makeObservation(input: ObsInput): ObservationRecord { + const eventMs = Date.parse(input.event_time); + return { + observation_id: ulid(Number.isFinite(eventMs) ? eventMs : 0), + entity_id: input.entity_id, + metric: input.metric, + value: input.value, + event_time: input.event_time, + ingest_time: input.ingest_time, + source_id: input.source_id, + feed_id: input.feed_id, + quality: input.quality ?? "good", + ...(input.unit !== undefined ? { unit: input.unit } : {}), + ...(input.is_correction !== undefined ? { is_correction: input.is_correction } : {}), + ...(input.source_sequence !== undefined ? { source_sequence: input.source_sequence } : {}), + ...(input.meta !== undefined ? { meta: input.meta } : {}), + }; +} + +/** String field access on a ParsedRecord row (missing → ""). */ +export function fieldStr(row: Record, key: string): string { + const v = row[key]; + return v === undefined ? "" : String(v); +} + +/** Numeric field access on a ParsedRecord row (missing/blank/NaN → null). */ +export function fieldNum(row: Record, key: string): number | null { + const v = row[key]; + if (v === undefined || v === "") return null; + const num = typeof v === "number" ? v : Number(v); + return Number.isFinite(num) ? num : null; +} + +/** Push an external id into `unmapped` once. */ +export function reportUnmapped(unmapped: string[], id: string): void { + if (id.length > 0 && !unmapped.includes(id)) unmapped.push(id); +} diff --git a/intelligence/packages/connectors/src/scheduler.ts b/intelligence/packages/connectors/src/scheduler.ts new file mode 100644 index 0000000..f6b438d --- /dev/null +++ b/intelligence/packages/connectors/src/scheduler.ts @@ -0,0 +1,130 @@ +// runOnce — one full discover→fetch→bronze→parse→normalize→silver pass for a +// connector — plus a cooperative setTimeout scheduler for --watch mode. + +import { createHash } from "node:crypto"; +import type { MedallionStore } from "@pact-tailor/store"; +import type { Connector, ConnectorCtx } from "./sdk.js"; + +export interface IngestReport { + connector_id: string; + /** Newly ingested artifacts (bronze puts) this pass. */ + artifacts: number; + /** Artifacts skipped because bronze already holds identical bytes (sha256). */ + skipped: number; + observations: number; + unmapped: string[]; + errors: string[]; +} + +/** + * Run one connector pass. Artifacts whose bytes (sha256) are already in + * bronze are skipped entirely — no re-put, no re-append — which makes + * repeated runs over the same fixtures/files idempotent. Per-artifact + * failures are recorded and do not abort the pass. + */ +export async function runOnce(connector: Connector, ctx: ConnectorCtx, store: MedallionStore): Promise { + const report: IngestReport = { + connector_id: connector.id, + artifacts: 0, + skipped: 0, + observations: 0, + unmapped: [], + errors: [], + }; + + let refs; + try { + refs = await connector.discover(ctx); + } catch (err) { + report.errors.push(`discover: ${(err as Error).message}`); + return report; + } + + const knownShas = new Set(store.bronze.list(connector.source_id).map((meta) => meta.sha256)); + + for (const ref of refs) { + try { + const raw = await connector.fetch(ref, ctx); + const sha256 = createHash("sha256").update(raw.body).digest("hex"); + if (knownShas.has(sha256)) { + report.skipped++; + continue; + } + store.bronze.put({ + source_id: connector.source_id, + url: ref.url, + fetched_at: raw.fetched_at, + http_status: raw.http_status, + ...(raw.content_type !== undefined ? { content_type: raw.content_type } : {}), + body: raw.body, + }); + knownShas.add(sha256); + const parsed = await connector.parse(raw); + const { observations, unmapped } = connector.normalize(parsed, ctx); + if (observations.length > 0) store.silver.append(observations); + report.artifacts++; + report.observations += observations.length; + for (const id of unmapped) { + if (!report.unmapped.includes(id)) report.unmapped.push(id); + } + } catch (err) { + report.errors.push(`${ref.name}: ${(err as Error).message}`); + } + } + return report; +} + +/** + * Cooperative per-connector loop: setTimeout at intervalMs plus uniform + * random jitter, rescheduled after each pass completes (no overlap per + * connector). Production swaps this for real workers + a queue — the + * Connector interface is the seam (STORAGE.md). + */ +export class Scheduler { + private readonly timers = new Map(); + private stopped = false; + + constructor( + private readonly store: MedallionStore, + private readonly makeCtx: (connector: Connector) => ConnectorCtx, + private readonly onReport?: (report: IngestReport) => void, + ) {} + + start(connectors: Connector[]): void { + this.stopped = false; + for (const connector of connectors) { + // First pass soon, spread by jitter so connectors don't stampede. + this.schedule(connector, Math.random() * connector.schedule.jitterMs); + } + } + + private schedule(connector: Connector, delayMs: number): void { + if (this.stopped) return; + const timer = setTimeout(() => { + void (async () => { + let report: IngestReport; + try { + report = await runOnce(connector, this.makeCtx(connector), this.store); + } catch (err) { + report = { + connector_id: connector.id, + artifacts: 0, + skipped: 0, + observations: 0, + unmapped: [], + errors: [`pass: ${(err as Error).message}`], + }; + } + this.onReport?.(report); + this.schedule(connector, connector.schedule.intervalMs + Math.random() * connector.schedule.jitterMs); + })(); + }, delayMs); + this.timers.set(connector.id, timer); + } + + stop(): void { + this.stopped = true; + for (const timer of this.timers.values()) clearTimeout(timer); + this.timers.clear(); + } +} diff --git a/intelligence/packages/connectors/src/sdk.ts b/intelligence/packages/connectors/src/sdk.ts new file mode 100644 index 0000000..caf7088 --- /dev/null +++ b/intelligence/packages/connectors/src/sdk.ts @@ -0,0 +1,95 @@ +// Connector SDK — the shared contract every source connector implements. +// +// Pipeline: discover() lists candidate artifacts, fetch() pulls verbatim bytes +// (bronze), parse() turns bytes into flat table rows, normalize() maps rows to +// canonical Observation envelopes (silver). The HttpGate seam makes fixture +// replay and live fetching the SAME code path — connectors never know which +// they are talking to. +// +// HONESTY: this workspace is built under blocked egress. Every connector is +// written against the source's DOCUMENTED wire format, exercised only through +// synthetic fixtures (fixture_provenance: synthetic-from-spec). Nothing in +// this package claims live verification; see verify-live.ts for the harness +// that a human runs to earn `verified_live` status. + +import type { EntityRecord, ObservationRecord, TelemetryFeedRecord } from "@pact-tailor/ontology"; +import type { HttpGate, MessageGate } from "./gate.js"; +import type { EntityIndex } from "./entity-index.js"; + +/** A discovered artifact: something fetchable that may yield observations. */ +export interface ArtifactRef { + url: string; + name: string; + meta?: Record; +} + +/** Verbatim fetched bytes plus fetch metadata — exactly what bronze stores. */ +export interface RawArtifact { + ref: ArtifactRef; + body: Buffer; + fetched_at: string; + http_status: number; + content_type?: string; +} + +/** One flat row extracted from a raw artifact, tagged with its table name. */ +export interface ParsedRecord { + table: string; + row: Record; +} + +/** + * Output of normalize(). `unmapped` lists the external ids (DUIDs, region ids, + * MMSIs, EICs, ...) that could not be resolved to an entity — their rows are + * DROPPED, never guessed at, and the ids are surfaced here so the gap is + * visible in every IngestReport. + */ +export interface NormalizedOutput { + observations: ObservationRecord[]; + unmapped: string[]; +} + +/** Result of a live verification attempt (see verify-live.ts). */ +export interface VerificationReport { + connector_id: string; + source_id: string; + ok: boolean; + error?: string; + checked_at: string; + notes?: string; +} + +/** Everything a connector is handed at runtime. */ +export interface ConnectorCtx { + /** HTTP seam: ReplayGate (fixtures) or LiveGate — same connector code path. */ + gate: HttpGate; + /** Websocket/stream seam, present for streaming connectors (aisstream). */ + messageGate?: MessageGate; + /** Wall-clock, injectable for deterministic tests. Returns ISO-8601 UTC. */ + now(): string; + /** External-id → entity_id resolution built from the registry + seed entities. */ + entityIndex: EntityIndex; + log(msg: string): void; +} + +export interface Connector { + /** Stable connector id; also names its fixtures dir: fixtures/{id}/. */ + id: string; + /** The source registry record this connector ingests from. */ + source_id: string; + /** Telemetry feeds this connector writes (silver partitions). */ + feeds: TelemetryFeedRecord[]; + schedule: { intervalMs: number; jitterMs: number }; + discover(ctx: ConnectorCtx): Promise; + fetch(ref: ArtifactRef, ctx: ConnectorCtx): Promise; + parse(raw: RawArtifact): Promise; + normalize(records: ParsedRecord[], ctx: ConnectorCtx): NormalizedOutput; + /** Optional custom live-verification probe (stubs report why they can't). */ + verify?(ctx: ConnectorCtx): Promise; + /** + * Connector-owned structural entities (e.g. weather stations) that are not + * part of the registry seed. The CLI merges these into the EntityIndex at + * load time; registry records win on entity_id collision. + */ + seedEntities?: EntityRecord[]; +} diff --git a/intelligence/packages/connectors/src/verify-live.ts b/intelligence/packages/connectors/src/verify-live.ts new file mode 100644 index 0000000..b3e2cae --- /dev/null +++ b/intelligence/packages/connectors/src/verify-live.ts @@ -0,0 +1,104 @@ +#!/usr/bin/env node +// verify:live — the live verification harness. +// +// For each connector: attempt ONE live discover → fetch → parse round trip +// (or the connector's own verify() probe), then write the results to +// var/verify-report.json and print a table. +// +// THIS HARNESS NEVER UPDATES THE YAML REGISTRIES. `verification_status` in +// data/registries/sources/*.yaml is upgraded to `verified_live` only by a +// human, after reviewing var/verify-report.json — honesty rule: nothing +// authored under blocked egress may claim live verification, and no tool may +// claim it automatically. + +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { EntityIndex } from "./entity-index.js"; +import { LiveGate, LiveMessageGate } from "./gate.js"; +import type { Connector, ConnectorCtx, VerificationReport } from "./sdk.js"; +import { allConnectors } from "./connectors/index.js"; + +const PKG_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const INTEL_ROOT = resolve(PKG_ROOT, "..", ".."); + +function makeCtx(connector: Connector): ConnectorCtx { + return { + gate: new LiveGate(), + messageGate: new LiveMessageGate(), + now: () => new Date().toISOString(), + // Verification exercises discover/fetch/parse only; no entity resolution. + entityIndex: new EntityIndex([]), + log: (msg: string) => console.error(`[${connector.id}] ${msg}`), + }; +} + +async function verifyOne(connector: Connector): Promise { + const ctx = makeCtx(connector); + const checkedAt = ctx.now(); + if (connector.verify) return connector.verify(ctx); + try { + const refs = await connector.discover(ctx); + const first = refs[0]; + if (first === undefined) { + return { + connector_id: connector.id, + source_id: connector.source_id, + ok: false, + error: "discover returned no artifacts", + checked_at: checkedAt, + }; + } + const raw = await connector.fetch(first, ctx); + const parsed = await connector.parse(raw); + return { + connector_id: connector.id, + source_id: connector.source_id, + ok: true, + checked_at: checkedAt, + notes: `${first.name}: ${raw.body.length} bytes, ${parsed.length} parsed record(s)`, + }; + } catch (err) { + return { + connector_id: connector.id, + source_id: connector.source_id, + ok: false, + error: (err as Error).message, + checked_at: checkedAt, + }; + } +} + +async function main(): Promise { + const reports: VerificationReport[] = []; + for (const connector of allConnectors) { + console.error(`verifying ${connector.id} ...`); + reports.push(await verifyOne(connector)); + } + + const varDir = join(INTEL_ROOT, "var"); + mkdirSync(varDir, { recursive: true }); + const reportPath = join(varDir, "verify-report.json"); + writeFileSync(reportPath, JSON.stringify(reports, null, 2) + "\n"); + + const headers = ["connector", "source", "ok", "detail"]; + const rows = reports.map((r) => [r.connector_id, r.source_id, r.ok ? "yes" : "NO", r.error ?? r.notes ?? "-"]); + const widths = headers.map((h, i) => Math.max(h.length, ...rows.map((row) => (row[i] ?? "").length))); + const line = (cells: string[]) => cells.map((c, i) => c.padEnd(widths[i] ?? 0)).join(" "); + console.log(line(headers)); + console.log(line(widths.map((w) => "-".repeat(w)))); + for (const row of rows) console.log(line(row)); + console.log(`\nreport written to ${reportPath}`); + console.log( + "NOTE: registry verification_status is NOT updated by this tool — a human reviews the report and edits the YAML.", + ); + return 0; +} + +main().then( + (code) => process.exit(code), + (err) => { + console.error(`verify-live: ${(err as Error).message}`); + process.exit(1); + }, +); diff --git a/intelligence/packages/connectors/src/zip.ts b/intelligence/packages/connectors/src/zip.ts new file mode 100644 index 0000000..8a1fe69 --- /dev/null +++ b/intelligence/packages/connectors/src/zip.ts @@ -0,0 +1,22 @@ +// NEMWEB artifacts are standard ZIP files containing a single CSV report. +// fflate's unzipSync does the work; we return the first real (non-directory) +// entry. + +import { unzipSync } from "fflate"; + +export interface ZipEntry { + name: string; + data: Buffer; +} + +/** Unzip and return the first non-directory entry. Throws on an empty archive. */ +export function unzipFirst(buf: Buffer): ZipEntry { + const entries = unzipSync(new Uint8Array(buf)); + for (const name of Object.keys(entries)) { + if (name.endsWith("/")) continue; // directory marker + const data = entries[name]; + if (data === undefined) continue; + return { name, data: Buffer.from(data) }; + } + throw new Error("zip: archive contains no file entries"); +} diff --git a/intelligence/packages/connectors/test/aemo-csv.test.mjs b/intelligence/packages/connectors/test/aemo-csv.test.mjs new file mode 100644 index 0000000..7ed03bf --- /dev/null +++ b/intelligence/packages/connectors/test/aemo-csv.test.mjs @@ -0,0 +1,78 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { parseAemoCsv, splitCsvLine, nemTimeToIso } from "../dist/index.js"; + +test("splitCsvLine: quoted fields, embedded commas, doubled quotes", () => { + assert.deepEqual(splitCsvLine('D,DISPATCH,PRICE,4,"2026/08/12 11:05:00",1'), [ + "D", + "DISPATCH", + "PRICE", + "4", + "2026/08/12 11:05:00", + "1", + ]); + assert.deepEqual(splitCsvLine('A,"a,b",c'), ["A", "a,b", "c"]); + assert.deepEqual(splitCsvLine('A,"say ""hi""",z'), ["A", 'say "hi"', "z"]); + assert.deepEqual(splitCsvLine("A,,B"), ["A", "", "B"]); +}); + +test("parseAemoCsv: I/D/C handling with multiple tables in one file", () => { + const csv = [ + "C,NEMP.WORLD,DISPATCHIS,AEMO,PUBLIC,2026/08/12,11:05:04,0000000000000001", + "I,DISPATCH,PRICE,4,SETTLEMENTDATE,RUNNO,REGIONID,RRP", + 'D,DISPATCH,PRICE,4,"2026/08/12 11:05:00",1,QLD1,84.50', + 'D,DISPATCH,PRICE,4,"2026/08/12 11:05:00",1,NSW1,92.10', + "I,DISPATCH,REGIONSUM,4,SETTLEMENTDATE,RUNNO,REGIONID,TOTALDEMAND", + 'D,DISPATCH,REGIONSUM,4,"2026/08/12 11:05:00",1,QLD1,6123.45', + 'C,"END OF REPORT",6', + ].join("\r\n"); + const tables = parseAemoCsv(csv); + assert.equal(tables.length, 2); + assert.equal(tables[0].table, "DISPATCH.PRICE"); + assert.equal(tables[0].rows.length, 2); + assert.deepEqual(tables[0].rows[0], { + SETTLEMENTDATE: "2026/08/12 11:05:00", + RUNNO: "1", + REGIONID: "QLD1", + RRP: "84.50", + }); + assert.equal(tables[1].table, "DISPATCH.REGIONSUM"); + assert.equal(tables[1].rows[0].TOTALDEMAND, "6123.45"); +}); + +test("parseAemoCsv: repeated I rows for the same table merge their D rows", () => { + const csv = [ + "I,T,X,1,A,B", + "D,T,X,1,1,2", + "I,T,X,1,A,B", + "D,T,X,1,3,4", + ].join("\n"); + const tables = parseAemoCsv(csv); + assert.equal(tables.length, 1); + assert.equal(tables[0].rows.length, 2); +}); + +test("parseAemoCsv: missing trailing values become empty strings", () => { + const tables = parseAemoCsv(["I,T,X,1,A,B,C", "D,T,X,1,only"].join("\n")); + assert.deepEqual(tables[0].rows[0], { A: "only", B: "", C: "" }); +}); + +test("parseAemoCsv: D row before its I row throws", () => { + assert.throws(() => parseAemoCsv("D,DISPATCH,PRICE,4,x"), /before its I row/); +}); + +test("nemTimeToIso: NEM time is UTC+10 fixed (no DST)", () => { + assert.equal(nemTimeToIso("2026/08/12 14:05:00"), "2026-08-12T04:05:00Z"); + assert.equal(nemTimeToIso("2026/08/12 11:05:00"), "2026-08-12T01:05:00Z"); + // dashboard variant with T separator and dashes + assert.equal(nemTimeToIso("2026-08-12T11:05:00"), "2026-08-12T01:05:00Z"); +}); + +test("nemTimeToIso: midnight and year rollovers", () => { + assert.equal(nemTimeToIso("2026/08/12 09:59:59"), "2026-08-11T23:59:59Z"); + assert.equal(nemTimeToIso("2026/01/01 04:00:00"), "2025-12-31T18:00:00Z"); +}); + +test("nemTimeToIso: malformed input throws", () => { + assert.throws(() => nemTimeToIso("12/08/2026 11:05"), /unrecognised NEM timestamp/); +}); diff --git a/intelligence/packages/connectors/test/connectors-aemo.test.mjs b/intelligence/packages/connectors/test/connectors-aemo.test.mjs new file mode 100644 index 0000000..7dc4f69 --- /dev/null +++ b/intelligence/packages/connectors/test/connectors-aemo.test.mjs @@ -0,0 +1,113 @@ +// AEMO connectors against their synthetic-from-spec fixtures, with an INLINE +// entity index (exact counts, exact mapping, exact unmapped reporting). +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + aemoNemwebDispatchis, + aemoNemwebDispatchScada, + aemoNemwebTradingis, + aemoVisNemSummary, +} from "../dist/index.js"; +import { entity, nemEntities, runPipeline, testCtx, FIXED_NOW } from "./helpers.mjs"; + +test("dispatchis: discover picks the newest ZIP from the dir listing", async () => { + const ctx = testCtx(aemoNemwebDispatchis.id, nemEntities); + const refs = await aemoNemwebDispatchis.discover(ctx); + assert.equal(refs.length, 1); + assert.equal(refs[0].name, "PUBLIC_DISPATCHIS_202608121105_0000000123456789.zip"); + assert.ok(refs[0].url.endsWith(refs[0].name)); +}); + +test("dispatchis: full pipeline — exact observations, corrections, unmapped", async () => { + const ctx = testCtx(aemoNemwebDispatchis.id, nemEntities); + const { parsed, observations, unmapped } = await runPipeline(aemoNemwebDispatchis, ctx); + assert.equal(parsed.length, 8); // 3 PRICE + 3 REGIONSUM + 2 INTERCONNECTORRES + + assert.equal(observations.length, 8); + assert.deepEqual(unmapped.sort(), ["MADEUP-X1", "SA1"]); + for (const obs of observations) { + assert.equal(obs.event_time, "2026-08-12T01:05:00Z"); // 11:05 NEM = 01:05Z + assert.equal(obs.ingest_time, FIXED_NOW); + assert.equal(obs.source_id, "source:aemo:nemweb-dispatchis"); + } + + const price = observations.filter((o) => o.feed_id === "feed:au-nem:dispatch:price"); + assert.equal(price.length, 3); + assert.deepEqual( + price.map((o) => [o.entity_id, o.value, o.is_correction]), + [ + ["region:au-nem:qld1", 84.5, false], + ["region:au-nem:nsw1", 92.1, false], + ["region:au-nem:nsw1", 95.75, true], // RUNNO=2 + INTERVENTION=1 re-run + ], + ); + assert.equal(price[0].source_sequence, "2026/08/12 11:05:00:1"); + assert.equal(price[2].source_sequence, "2026/08/12 11:05:00:2"); + + const regionsum = observations.filter((o) => o.feed_id === "feed:au-nem:dispatch:regionsum"); + assert.equal(regionsum.length, 4); // QLD1+NSW1 × (demand, generation); SA1 dropped + const qldDemand = regionsum.find((o) => o.entity_id === "region:au-nem:qld1" && o.metric === "grid.demand.mw"); + assert.equal(qldDemand.value, 6123.45); + const qldGen = regionsum.find((o) => o.entity_id === "region:au-nem:qld1" && o.metric === "grid.generation.mw"); + assert.equal(qldGen.value, 5900.1); // DISPATCHABLEGENERATION preferred over AVAILABLEGENERATION + + const flows = observations.filter((o) => o.feed_id === "feed:au-nem:dispatch:interconnectorres"); + assert.equal(flows.length, 1); + assert.equal(flows[0].entity_id, "intercon:au-nem:qni"); // byDuid("NSW1-QLD1") + assert.equal(flows[0].metric, "intercon.flow.mw"); + assert.equal(flows[0].value, 350); +}); + +test("dispatch-scada: DUID mapping to station entities, unmapped DUID reported", async () => { + const entities = [ + entity("gen:au-nem:gladstone-ps", "Generator", { + aemo_duid: ["GSTONE1", "GSTONE2", "GSTONE3", "GSTONE4", "GSTONE5", "GSTONE6"], + }), + entity("gen:au-nem:callide-b", "Generator", { aemo_duid: ["CALL_B_1", "CALL_B_2"] }), + ]; + const ctx = testCtx(aemoNemwebDispatchScada.id, entities); + const { observations, unmapped } = await runPipeline(aemoNemwebDispatchScada, ctx); + assert.equal(observations.length, 7); // GSTONE1..6 + CALL_B_1; NOMAP_1 dropped + assert.deepEqual(unmapped, ["NOMAP_1"]); + for (const obs of observations) { + assert.equal(obs.metric, "power.output.mw"); + assert.equal(obs.feed_id, "feed:au-nem:dispatch:scada"); + assert.equal(obs.event_time, "2026-08-12T01:05:00Z"); + } + const gladstone = observations.filter((o) => o.entity_id === "gen:au-nem:gladstone-ps"); + assert.equal(gladstone.length, 6); + assert.deepEqual(new Set(gladstone.map((o) => o.meta.duid)).size, 6); // unit kept in meta + const callide = observations.find((o) => o.entity_id === "gen:au-nem:callide-b"); + assert.equal(callide.value, 312.4); +}); + +test("tradingis: TRADING.PRICE on its own feed, distinct from dispatch price", async () => { + const ctx = testCtx(aemoNemwebTradingis.id, nemEntities); + const { observations, unmapped } = await runPipeline(aemoNemwebTradingis, ctx); + assert.equal(observations.length, 2); + assert.deepEqual(unmapped, ["SA1"]); + assert.deepEqual( + observations.map((o) => [o.feed_id, o.entity_id, o.metric, o.value]), + [ + ["feed:au-nem:trading:price", "region:au-nem:qld1", "market.price.energy", 86.2], + ["feed:au-nem:trading:price", "region:au-nem:nsw1", "market.price.energy", 93.4], + ], + ); + assert.equal(observations[0].source_id, "source:aemo:nemweb-tradingis"); +}); + +test("vis-nem-summary: dashboard JSON → demand/generation/price per region", async () => { + const ctx = testCtx(aemoVisNemSummary.id, nemEntities); + const { parsed, observations, unmapped } = await runPipeline(aemoVisNemSummary, ctx); + assert.equal(parsed.length, 3); // QLD1, NSW1, SA1 rows + assert.equal(observations.length, 6); // 3 metrics × 2 mapped regions + assert.deepEqual(unmapped, ["SA1"]); + const qld = observations.filter((o) => o.entity_id === "region:au-nem:qld1"); + assert.equal(qld.find((o) => o.metric === "grid.demand.mw").value, 6123.45); + // SCHEDULEDGENERATION + SEMISCHEDULEDGENERATION + assert.ok(Math.abs(qld.find((o) => o.metric === "grid.generation.mw").value - 5900.1) < 1e-9); + assert.equal(qld.find((o) => o.metric === "market.price.energy").value, 84.5); + // dashboard "2026-08-12T11:05:00" NEM time converts like NEMWEB timestamps + assert.equal(qld[0].event_time, "2026-08-12T01:05:00Z"); + assert.equal(qld[0].feed_id, "feed:au-nem:vis-nem-summary"); +}); diff --git a/intelligence/packages/connectors/test/connectors-stream.test.mjs b/intelligence/packages/connectors/test/connectors-stream.test.mjs new file mode 100644 index 0000000..a3543d7 --- /dev/null +++ b/intelligence/packages/connectors/test/connectors-stream.test.mjs @@ -0,0 +1,96 @@ +// aisstream (MessageGate replay) and bom-observations, with inline entities — +// the vessel entities deliberately carry the synthetic exemplar MMSIs +// (503000001..503000003) so these tests do not depend on the data/ lane. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { aisstream, bomObservations, aisTimeToIso, GLADSTONE_BBOX } from "../dist/index.js"; +import { entity, runPipeline, testCtx } from "./helpers.mjs"; + +const vesselEntities = [ + entity("vessel:au:synthetic-demo-1", "Vessel", { mmsi: "503000001" }, { synthetic: true }), + entity("vessel:au:synthetic-demo-2", "Vessel", { mmsi: "503000002" }, { synthetic: true }), + entity("vessel:au:synthetic-demo-3", "Vessel", { mmsi: "503000003" }, { synthetic: true }), +]; + +test("aisTimeToIso: Go-style aisstream timestamps", () => { + assert.equal(aisTimeToIso("2026-08-12 01:00:00.000000000 +0000 UTC"), "2026-08-12T01:00:00Z"); + assert.equal(aisTimeToIso("2026-08-12 01:05:00 +0000 UTC"), "2026-08-12T01:05:00Z"); + assert.throws(() => aisTimeToIso("yesterday"), /unrecognised time_utc/); +}); + +test("aisstream: bbox constant covers Gladstone harbour ([lat,lon] order)", () => { + const [[lat1, lon1], [lat2, lon2]] = GLADSTONE_BBOX; + assert.ok(lat1 < -23.8 && lat2 > -23.9); // straddles the harbour latitude + assert.ok(lon1 < 151.2 && lon2 > 151.3); +}); + +test("aisstream: replayed window → position/speed/draught obs, unmapped MMSIs dropped", async () => { + const ctx = testCtx(aisstream.id, vesselEntities); + const { raws, parsed, observations, unmapped } = await runPipeline(aisstream, ctx); + assert.equal(raws.length, 1); + assert.equal(raws[0].content_type, "application/x-ndjson"); + assert.equal(parsed.length, 20); // 18 PositionReports + 2 ShipStaticData + + // 15 mapped PositionReports × (position + speed) + 1 mapped ShipStaticData draught + assert.equal(observations.length, 31); + assert.deepEqual(unmapped, ["999999999"]); + + const positions = observations.filter((o) => o.metric === "vessel.position"); + assert.equal(positions.length, 15); + const first = positions.find((o) => o.entity_id === "vessel:au:synthetic-demo-1"); + assert.equal(first.event_time, "2026-08-12T01:00:00Z"); + assert.deepEqual(first.value, { lat: -23.82, lon: 151.25, speed_kn: 8.2, heading_deg: 121 }); + assert.equal(first.feed_id, "feed:maritime:aisstream:gladstone"); + assert.equal(first.source_id, "source:aisstream:websocket"); + + // TrueHeading 511 (unavailable sentinel) must not become a heading + const demo2 = positions.find((o) => o.entity_id === "vessel:au:synthetic-demo-2"); + assert.equal(demo2.value.heading_deg, undefined); + assert.equal(demo2.value.speed_kn, 0.1); + + const speeds = observations.filter((o) => o.metric === "vessel.speed.kn"); + assert.equal(speeds.length, 15); + + const draughts = observations.filter((o) => o.metric === "vessel.draught.m"); + assert.equal(draughts.length, 1); // mapped ShipStaticData only + assert.equal(draughts[0].value, 10.5); + assert.equal(draughts[0].entity_id, "vessel:au:synthetic-demo-1"); +}); + +test("aisstream: fetch without a MessageGate fails loudly", async () => { + const ctx = { ...testCtx(aisstream.id, vesselEntities), messageGate: undefined }; + await assert.rejects(() => aisstream.fetch({ url: "wss://x", name: "w" }, ctx), /messageGate is required/); +}); + +test("bom-observations: seeded Sensor entity → temp + wind observations", async () => { + // The connector exports its own Sensor seed; merging it is exactly what the + // CLI does. wind km/h → m/s. + const ctx = testCtx(bomObservations.id, bomObservations.seedEntities); + const { observations, unmapped } = await runPipeline(bomObservations, ctx); + assert.equal(observations.length, 6); // 3 rows × (temp, wind) + assert.deepEqual(unmapped, []); + for (const obs of observations) { + assert.equal(obs.entity_id, "sensor:au-qld:gladstone-aws"); + assert.equal(obs.feed_id, "feed:weather:bom:qld-obs"); + } + const temps = observations.filter((o) => o.metric === "weather.temp.c"); + assert.deepEqual(temps.map((o) => o.value), [18.4, 18.1, 17.8]); + assert.equal(temps[0].event_time, "2026-08-12T01:30:00Z"); // aifstime_utc is already UTC + const winds = observations.filter((o) => o.metric === "weather.wind.speed_ms"); + assert.deepEqual(winds.map((o) => o.value), [5.28, 4.72, 4.17]); // 19/17/15 km/h → m/s +}); + +test("bom-observations: station absent from the index → rows DROPPED and wmo reported", async () => { + const ctx = testCtx(bomObservations.id, []); // seed NOT merged + const { observations, unmapped } = await runPipeline(bomObservations, ctx); + assert.equal(observations.length, 0); + assert.deepEqual(unmapped, ["94381"]); +}); + +test("bom-observations: seedEntities is a valid connector-owned Sensor record", () => { + assert.equal(bomObservations.seedEntities.length, 1); + const seed = bomObservations.seedEntities[0]; + assert.equal(seed.entity_id, "sensor:au-qld:gladstone-aws"); + assert.equal(seed.entity_type, "Sensor"); + assert.deepEqual(seed.sources, ["source:bom:observations"]); +}); diff --git a/intelligence/packages/connectors/test/connectors-stubs.test.mjs b/intelligence/packages/connectors/test/connectors-stubs.test.mjs new file mode 100644 index 0000000..d6641d3 --- /dev/null +++ b/intelligence/packages/connectors/test/connectors-stubs.test.mjs @@ -0,0 +1,75 @@ +// Stub connectors (entsoe-transparency, eia-v2): replay path fully works, +// live path honestly refuses (registration/api-key required). +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { entsoeTransparency, eiaV2, xmlBlocks, xmlText } from "../dist/index.js"; +import { entity, runPipeline, testCtx } from "./helpers.mjs"; + +const liveCtx = (base) => ({ + ...base, + gate: { + kind: "live", + get: async () => { + throw new Error("network must not be touched by stub connectors in live mode"); + }, + }, +}); + +test("xml scanner: blocks and text extraction (dotted tags, attributes)", () => { + const xml = "onetwodot"; + assert.deepEqual(xmlBlocks(xml, "B"), ["one", "two"]); + assert.equal(xmlText(xml, "c.d"), "dot"); + assert.equal(xmlText(xml, "missing"), undefined); +}); + +test("entsoe-transparency: A75 replay parse+normalize; EIC mapping; unmapped zone", async () => { + const entities = [entity("grid:eu-test:zone", "Grid", { entsoe_eic: "10Y-TEST-ZONE--X" })]; + const ctx = testCtx(entsoeTransparency.id, entities); + const { parsed, observations, unmapped } = await runPipeline(entsoeTransparency, ctx); + assert.equal(parsed.length, 4); // 2 points × 2 TimeSeries + assert.equal(observations.length, 2); // only the mapped zone survives + assert.deepEqual(unmapped, ["10Y-UNMAPPED---Z"]); + assert.deepEqual( + observations.map((o) => [o.metric, o.value, o.event_time, o.meta.psr_type]), + [ + ["grid.generation.mw", 640, "2026-08-12T00:00:00Z", "B05"], + ["grid.generation.mw", 655, "2026-08-12T01:00:00Z", "B05"], // position 2 at PT60M + ], + ); + assert.equal(observations[0].feed_id, "feed:eu:entsoe:actual-generation"); +}); + +test("entsoe-transparency: live discover returns [] and live fetch throws (no token, no fetch)", async () => { + const ctx = liveCtx(testCtx(entsoeTransparency.id, [])); + assert.deepEqual(await entsoeTransparency.discover(ctx), []); + await assert.rejects(() => entsoeTransparency.fetch({ url: "https://x", name: "x" }, ctx), /securityToken/); + const report = await entsoeTransparency.verify(ctx); + assert.equal(report.ok, false); + assert.match(report.error, /securityToken/); +}); + +test("eia-v2: region-data replay parse+normalize; BA mapping via properties.eia_ba_code", async () => { + const entities = [entity("grid:us-test:demo-ba", "Grid", {}, { eia_ba_code: "DEMO" })]; + const ctx = testCtx(eiaV2.id, entities); + const { parsed, observations, unmapped } = await runPipeline(eiaV2, ctx); + assert.equal(parsed.length, 3); + assert.equal(observations.length, 2); + assert.deepEqual(unmapped, ["NOBA"]); + assert.deepEqual( + observations.map((o) => [o.entity_id, o.metric, o.value, o.event_time]), + [ + ["grid:us-test:demo-ba", "grid.demand.mw", 21450, "2026-08-12T00:00:00Z"], + ["grid:us-test:demo-ba", "grid.demand.mw", 20980, "2026-08-12T01:00:00Z"], + ], + ); + assert.equal(observations[0].feed_id, "feed:us:eia:demand"); + assert.equal(observations[0].meta.source_units, "megawatthours"); +}); + +test("eia-v2: live discover returns [] and live fetch throws (api key required)", async () => { + const ctx = liveCtx(testCtx(eiaV2.id, [])); + assert.deepEqual(await eiaV2.discover(ctx), []); + await assert.rejects(() => eiaV2.fetch({ url: "https://x", name: "x" }, ctx), /api_key/); + const report = await eiaV2.verify(ctx); + assert.equal(report.ok, false); +}); diff --git a/intelligence/packages/connectors/test/e2e.test.mjs b/intelligence/packages/connectors/test/e2e.test.mjs new file mode 100644 index 0000000..42bea0f --- /dev/null +++ b/intelligence/packages/connectors/test/e2e.test.mjs @@ -0,0 +1,82 @@ +// End-to-end: runOnce → bronze + silver in a temp MedallionStore, idempotent +// re-runs, and the pt-ingest CLI over every connector's fixtures. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { MedallionStore } from "@pact-tailor/store"; +import { aemoNemwebDispatchis, runOnce } from "../dist/index.js"; +import { nemEntities, testCtx, PKG_ROOT } from "./helpers.mjs"; + +test("runOnce: discover→fetch→bronze→parse→normalize→silver, idempotent re-run", async () => { + const varDir = mkdtempSync(join(tmpdir(), "pt-e2e-")); + const store = new MedallionStore(varDir); + const ctx = testCtx(aemoNemwebDispatchis.id, nemEntities); + + const first = await runOnce(aemoNemwebDispatchis, ctx, store); + assert.equal(first.connector_id, "aemo-nemweb-dispatchis"); + assert.equal(first.artifacts, 1); + assert.equal(first.skipped, 0); + assert.equal(first.observations, 8); + assert.deepEqual(first.unmapped.sort(), ["MADEUP-X1", "SA1"]); + assert.deepEqual(first.errors, []); + + // bronze holds the verbatim ZIP + const bronze = store.bronze.list("source:aemo:nemweb-dispatchis"); + assert.equal(bronze.length, 1); + assert.ok(bronze[0].url.endsWith("PUBLIC_DISPATCHIS_202608121105_0000000123456789.zip")); + assert.equal(bronze[0].http_status, 200); + const body = store.bronze.get(bronze[0].source_id, bronze[0].artifact_id).body; + assert.deepEqual(body.subarray(0, 2), Buffer.from("PK")); // still a ZIP, byte-for-byte + + // silver holds the normalized observations, partitioned by feed + assert.equal(store.silver.readFeed("feed:au-nem:dispatch:price").length, 3); + assert.equal(store.silver.readFeed("feed:au-nem:dispatch:regionsum").length, 4); + assert.equal(store.silver.readFeed("feed:au-nem:dispatch:interconnectorres").length, 1); + + // re-run: identical bytes are sha-skipped — no duplicate bronze, no duplicate silver + const second = await runOnce(aemoNemwebDispatchis, ctx, store); + assert.equal(second.artifacts, 0); + assert.equal(second.skipped, 1); + assert.equal(second.observations, 0); + assert.deepEqual(second.errors, []); + assert.equal(store.bronze.list("source:aemo:nemweb-dispatchis").length, 1); + assert.equal(store.silver.readFeed("feed:au-nem:dispatch:price").length, 3); +}); + +test("pt-ingest ingest --replay: runs every connector over fixtures, exit 0", () => { + const varDir = mkdtempSync(join(tmpdir(), "pt-cli-")); + // --data points at an empty temp dir: hermetic — only connector seedEntities + // resolve (BOM), everything else reports unmapped. Exit must still be 0. + const dataDir = mkdtempSync(join(tmpdir(), "pt-cli-data-")); + const stdout = execFileSync(process.execPath, [ + join(PKG_ROOT, "dist", "cli.js"), + "ingest", + "--replay", + "--var", + varDir, + "--data", + dataDir, + ]).toString(); + assert.match(stdout, /aemo-nemweb-dispatchis/); + assert.match(stdout, /eia-v2/); + + const store = new MedallionStore(varDir); + // every non-stream connector left exactly one bronze artifact + for (const sourceId of [ + "source:aemo:nemweb-dispatchis", + "source:aemo:nemweb-dispatch-scada", + "source:aemo:nemweb-tradingis", + "source:aemo:vis-nem-summary", + "source:aisstream:websocket", + "source:bom:observations", + "source:entsoe:transparency-platform", + "source:eia:api-v2", + ]) { + assert.equal(store.bronze.list(sourceId).length, 1, `bronze for ${sourceId}`); + } + // BOM's seed entity resolves without any registry data + assert.equal(store.silver.readFeed("feed:weather:bom:qld-obs").length, 6); +}); diff --git a/intelligence/packages/connectors/test/gate.test.mjs b/intelligence/packages/connectors/test/gate.test.mjs new file mode 100644 index 0000000..895489d --- /dev/null +++ b/intelligence/packages/connectors/test/gate.test.mjs @@ -0,0 +1,51 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { ReplayGate, ReplayMessageGate, REPLAY_UNMATCHED_STATUS } from "../dist/index.js"; + +function fixtureDirWith(exchanges) { + const dir = mkdtempSync(join(tmpdir(), "pt-gate-")); + mkdirSync(join(dir, "exchanges"), { recursive: true }); + exchanges.forEach((exchange, i) => { + writeFileSync(join(dir, "exchanges", `0${i}-x.json`), JSON.stringify(exchange)); + }); + return dir; +} + +test("ReplayGate: exact URL match wins over prefix; longest prefix otherwise", async () => { + const dir = fixtureDirWith([ + { url_pattern: "https://x.test/reports/", status: 200, content_type: "text/html", body_base64: Buffer.from("listing").toString("base64") }, + { url_pattern: "https://x.test/reports/a.zip", status: 200, body_base64: Buffer.from("exact").toString("base64") }, + { url_pattern: "https://x.test/", status: 200, body_base64: Buffer.from("root").toString("base64") }, + ]); + const gate = new ReplayGate(dir); + assert.equal(gate.kind, "replay"); + assert.equal((await gate.get("https://x.test/reports/a.zip")).body.toString(), "exact"); + // no exact match → longest prefix (the /reports/ listing, not the root) + const prefixed = await gate.get("https://x.test/reports/b.zip"); + assert.equal(prefixed.body.toString(), "listing"); + assert.equal(prefixed.content_type, "text/html"); + assert.equal((await gate.get("https://x.test/other")).body.toString(), "root"); +}); + +test("ReplayGate: unmatched URL surfaces as status 599, not a silent pass", async () => { + const gate = new ReplayGate(fixtureDirWith([])); + const res = await gate.get("https://nothing.test/at-all"); + assert.equal(res.status, REPLAY_UNMATCHED_STATUS); + assert.equal(res.body.length, 0); +}); + +test("ReplayMessageGate: replays JSONL lines in order with zero delay; missing file → []", async () => { + const dir = mkdtempSync(join(tmpdir(), "pt-mgate-")); + const file = join(dir, "messages.jsonl"); + writeFileSync(file, '{"a":1}\n{"a":2}\n\n{"a":3}\n'); + const gate = new ReplayMessageGate(file); + const all = await gate.collect("wss://ignored", { APIKey: "" }); + assert.deepEqual(all.map((b) => JSON.parse(b.toString()).a), [1, 2, 3]); + const capped = await gate.collect("wss://ignored", {}, { maxMessages: 2 }); + assert.equal(capped.length, 2); + const empty = new ReplayMessageGate(join(dir, "absent.jsonl")); + assert.deepEqual(await empty.collect("wss://ignored", {}), []); +}); diff --git a/intelligence/packages/connectors/test/helpers.mjs b/intelligence/packages/connectors/test/helpers.mjs new file mode 100644 index 0000000..07468bc --- /dev/null +++ b/intelligence/packages/connectors/test/helpers.mjs @@ -0,0 +1,53 @@ +// Shared test scaffolding. Tests use INLINE entity records (not the data/ +// lane, which a concurrent agent owns) so they are hermetic and exact. +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { EntityIndex, ReplayGate, ReplayMessageGate } from "../dist/index.js"; + +export const PKG_ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); +export const fixturesDir = (connectorId) => join(PKG_ROOT, "fixtures", connectorId); + +export const entity = (entity_id, entity_type, external_ids = {}, properties = {}) => ({ + entity_id, + entity_type, + name: entity_id, + data_class: "structural", + observability: "KNOWN_LIVE", + sources: ["source:test:seed"], + external_ids, + properties, +}); + +export const FIXED_NOW = "2026-08-12T01:10:00Z"; + +export const testCtx = (connectorId, entities, now = FIXED_NOW) => { + const dir = fixturesDir(connectorId); + return { + gate: new ReplayGate(dir), + messageGate: new ReplayMessageGate(join(dir, "messages.jsonl")), + now: () => now, + entityIndex: new EntityIndex(entities), + log: () => {}, + }; +}; + +/** Inline NEM entities matching the synthetic fixtures (regions + QNI). */ +export const nemEntities = [ + entity("region:au-nem:qld1", "GridRegion", { aemo_region_id: "QLD1" }), + entity("region:au-nem:nsw1", "GridRegion", { aemo_region_id: "NSW1" }), + entity("intercon:au-nem:qni", "Interconnector", { aemo_duid: ["NSW1-QLD1"] }), +]; + +/** Full discover→fetch→parse→normalize pass without a store. */ +export async function runPipeline(connector, ctx) { + const refs = await connector.discover(ctx); + const raws = []; + let parsed = []; + for (const ref of refs) { + const raw = await connector.fetch(ref, ctx); + raws.push(raw); + parsed = parsed.concat(await connector.parse(raw)); + } + const { observations, unmapped } = connector.normalize(parsed, ctx); + return { refs, raws, parsed, observations, unmapped }; +} diff --git a/intelligence/packages/connectors/test/zip.test.mjs b/intelligence/packages/connectors/test/zip.test.mjs new file mode 100644 index 0000000..c8a8ab2 --- /dev/null +++ b/intelligence/packages/connectors/test/zip.test.mjs @@ -0,0 +1,19 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { zipSync } from "fflate"; +import { unzipFirst } from "../dist/index.js"; + +test("zip: fflate zipSync round-trips through unzipFirst", () => { + const csv = "I,DISPATCH,PRICE,4,SETTLEMENTDATE,RRP\nD,DISPATCH,PRICE,4,\"2026/08/12 11:05:00\",84.50\n"; + const zipped = zipSync({ "PUBLIC_TEST.CSV": new TextEncoder().encode(csv) }); + const entry = unzipFirst(Buffer.from(zipped)); + assert.equal(entry.name, "PUBLIC_TEST.CSV"); + assert.equal(entry.data.toString("utf8"), csv); +}); + +test("zip: first non-directory entry wins; empty archive throws", () => { + const zipped = zipSync({ "sub/": new Uint8Array(0), "sub/inner.txt": new TextEncoder().encode("x") }); + const entry = unzipFirst(Buffer.from(zipped)); + assert.equal(entry.name, "sub/inner.txt"); + assert.throws(() => unzipFirst(Buffer.from(zipSync({}))), /no file entries/); +}); From bfe7727b365a309209abcce30d460c8288dd0b5f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 02:05:38 +0000 Subject: [PATCH 13/16] feat(intelligence): derived-intelligence rules (phases 10-13, 16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deterministic, versioned rules — the system's ONLY producer of class-C data: smelter-utilisation (grid-demand proxy + SUPPLIES edge evidence), cargo-inference (terminal commodity profile; never emits without a LOADS_AT edge), anomaly-detection (3-sigma over event-time history), gap-synthesis (proposes gap records for unobserved strategic entities). Every record is schema-validated on emit — a rule producing invalid intelligence throws. React mode re-runs anomaly detection per state delta, debounced. 15 tests. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014yVGqBtvM1zNAd9tvUVNzm --- intelligence/packages/intel/package.json | 6 +- intelligence/packages/intel/src/index.ts | 189 ++++++++++- .../intel/src/rules/anomaly-detection.ts | 99 ++++++ .../intel/src/rules/cargo-inference.ts | 112 +++++++ .../packages/intel/src/rules/gap-synthesis.ts | 56 ++++ .../packages/intel/src/rules/index.ts | 19 ++ .../intel/src/rules/smelter-utilisation.ts | 118 +++++++ .../packages/intel/src/rules/types.ts | 48 +++ .../packages/intel/test/engine.test.mjs | 136 ++++++++ .../packages/intel/test/rules.test.mjs | 299 ++++++++++++++++++ 10 files changed, 1078 insertions(+), 4 deletions(-) create mode 100644 intelligence/packages/intel/src/rules/anomaly-detection.ts create mode 100644 intelligence/packages/intel/src/rules/cargo-inference.ts create mode 100644 intelligence/packages/intel/src/rules/gap-synthesis.ts create mode 100644 intelligence/packages/intel/src/rules/index.ts create mode 100644 intelligence/packages/intel/src/rules/smelter-utilisation.ts create mode 100644 intelligence/packages/intel/src/rules/types.ts create mode 100644 intelligence/packages/intel/test/engine.test.mjs create mode 100644 intelligence/packages/intel/test/rules.test.mjs diff --git a/intelligence/packages/intel/package.json b/intelligence/packages/intel/package.json index 9eede4a..2a8db1a 100644 --- a/intelligence/packages/intel/package.json +++ b/intelligence/packages/intel/package.json @@ -13,11 +13,13 @@ "node": ">=20" }, "scripts": { - "build": "tsc -p tsconfig.json" + "build": "tsc -p tsconfig.json", + "test": "npm run build && node --test \"test/*.test.mjs\"" }, "dependencies": { "@pact-tailor/graph": "*", "@pact-tailor/ontology": "*", - "@pact-tailor/state": "*" + "@pact-tailor/state": "*", + "@pact-tailor/store": "*" } } diff --git a/intelligence/packages/intel/src/index.ts b/intelligence/packages/intel/src/index.ts index 5cd3b3d..89ee5ff 100644 --- a/intelligence/packages/intel/src/index.ts +++ b/intelligence/packages/intel/src/index.ts @@ -1,2 +1,187 @@ -// @pact-tailor/intel — derived intelligence rules. Populated in the intel commit. -export {}; +// @pact-tailor/intel — deterministic, versioned rules producing evidence-linked +// InferenceRecords (class C). The IntelEngine is the ONLY producer of class-C +// data in the system: every record it lets out has been schema-validated (a +// rule that emits invalid intelligence is a bug, and the engine throws). +import { EventEmitter } from "node:events"; +import { + createValidators, + type InferenceRecord, + type ObservabilityGapRecord, + type Validators, +} from "@pact-tailor/ontology"; +import type { KnowledgeGraph } from "@pact-tailor/graph"; +import type { StateEngine, StateDelta } from "@pact-tailor/state"; +import type { MedallionStore } from "@pact-tailor/store"; +import { ALL_RULES, anomaliesForKey } from "./rules/index.js"; +import type { IntelRule, RuleContext } from "./rules/types.js"; + +export { + ALL_RULES, + smelterUtilisation, + cargoInference, + anomalyDetection, + anomaliesForKey, + gapSynthesis, + methodOf, + MIN_POINTS, + SUGGESTED_METRIC, + type IntelRule, + type RuleContext, +} from "./rules/index.js"; + +export interface IntelEngineOptions { + graph: KnowledgeGraph; + state: StateEngine; + gaps: readonly ObservabilityGapRecord[]; + /** Rule set override (default: ALL_RULES). */ + rules?: readonly IntelRule[]; + /** Debounce for react-mode anomaly re-runs, ms (default 250). */ + reactDebounceMs?: number; +} + +const INFERENCE_EVENT = "inference"; + +export class IntelEngine { + private readonly graph: KnowledgeGraph; + private readonly state: StateEngine; + private readonly gaps: readonly ObservabilityGapRecord[]; + private readonly rules: readonly IntelRule[]; + private readonly validators: Validators; + private readonly emitter = new EventEmitter(); + private readonly reactDebounceMs: number; + + /** Every validated record produced by this engine, in production order. */ + private readonly inferences: InferenceRecord[] = []; + /** How many of `inferences` have already been appended to gold. */ + private persisted = 0; + /** Engine-scoped id counters, keyed `${day}:${slug}` — unique across runs. */ + private readonly idCounters = new Map(); + /** React-mode debounce timers keyed `${entity_id} ${metric}`. */ + private readonly reactTimers = new Map(); + private detachBus: (() => void) | null = null; + + constructor(opts: IntelEngineOptions) { + this.graph = opts.graph; + this.state = opts.state; + this.gaps = opts.gaps; + this.rules = opts.rules ?? ALL_RULES; + this.reactDebounceMs = opts.reactDebounceMs ?? 250; + this.validators = createValidators(); + } + + private makeContext(now: string): RuleContext { + const counters = this.idCounters; + return { + graph: this.graph, + state: this.state, + gaps: this.gaps, + now: () => now, + mintId: (slug: string): string => { + const day = now.slice(0, 10); + const key = `${day}:${slug}`; + const n = (counters.get(key) ?? 0) + 1; + counters.set(key, n); + return `infer:${day}:${slug}-${n}`; + }, + }; + } + + /** Validate, record, and announce a batch of rule output. Throws on invalid. */ + private accept(rule: Pick, records: InferenceRecord[]): InferenceRecord[] { + for (const record of records) { + const result = this.validators.inference(record); + if (!result.ok) { + throw new Error( + `intel: rule ${rule.name}@${rule.version} emitted a schema-invalid inference ` + + `(${record.inference_id}): ${result.errors.join("; ")}`, + ); + } + } + for (const record of records) { + this.inferences.push(record); + this.emitter.emit(INFERENCE_EVENT, record); + } + return records; + } + + /** Run every rule once. Returns the records produced by THIS call. */ + runAll(now: string = new Date().toISOString()): InferenceRecord[] { + const produced: InferenceRecord[] = []; + const ctx = this.makeContext(now); + for (const rule of this.rules) { + produced.push(...this.accept(rule, rule.run(ctx))); + } + return produced; + } + + /** Run one rule by name. Throws on unknown rule name. */ + runRule(name: string, now: string = new Date().toISOString()): InferenceRecord[] { + const rule = this.rules.find((r) => r.name === name); + if (rule === undefined) throw new Error(`intel: unknown rule "${name}"`); + return this.accept(rule, rule.run(this.makeContext(now))); + } + + /** All records produced so far, in production order. */ + all(): InferenceRecord[] { + return [...this.inferences]; + } + + /** Records whose claim subject is the given entity. */ + byEntity(entityId: string): InferenceRecord[] { + return this.inferences.filter((i) => i.claim_structured.subject === entityId); + } + + byId(inferenceId: string): InferenceRecord | null { + return this.inferences.find((i) => i.inference_id === inferenceId) ?? null; + } + + /** Append records not yet persisted to the gold inference log. Returns the count appended. */ + persistTo(store: MedallionStore): number { + const fresh = this.inferences.slice(this.persisted); + for (const record of fresh) store.gold.appendInference(record); + this.persisted = this.inferences.length; + return fresh.length; + } + + /** Subscribe to every record the engine accepts (runAll, runRule, react mode). */ + onInference(cb: (record: InferenceRecord) => void): () => void { + this.emitter.on(INFERENCE_EVENT, cb); + return () => { + this.emitter.off(INFERENCE_EVENT, cb); + }; + } + + /** + * React mode: subscribe to state deltas and re-run anomaly-detection for the + * affected (entity, metric) key only, debounced per key. Returns the detach + * function (also clears pending timers). + */ + attach(): () => void { + if (this.detachBus) return () => this.detach(); + this.detachBus = this.state.onDelta((delta: StateDelta) => { + const { entity_id, metric } = delta.state; + const key = `${entity_id} ${metric}`; + const pending = this.reactTimers.get(key); + if (pending !== undefined) clearTimeout(pending); + const timer = setTimeout(() => { + this.reactTimers.delete(key); + const ctx = this.makeContext(new Date().toISOString()); + const records = anomaliesForKey(ctx, entity_id, metric); + this.accept({ name: "anomaly-detection", version: "1.0.0" }, records); + }, this.reactDebounceMs); + timer.unref?.(); + this.reactTimers.set(key, timer); + }); + return () => this.detach(); + } + + /** Detach react mode and cancel pending debounced re-runs. */ + detach(): void { + if (this.detachBus) { + this.detachBus(); + this.detachBus = null; + } + for (const timer of this.reactTimers.values()) clearTimeout(timer); + this.reactTimers.clear(); + } +} diff --git a/intelligence/packages/intel/src/rules/anomaly-detection.ts b/intelligence/packages/intel/src/rules/anomaly-detection.ts new file mode 100644 index 0000000..53e0f51 --- /dev/null +++ b/intelligence/packages/intel/src/rules/anomaly-detection.ts @@ -0,0 +1,99 @@ +// anomaly-detection@1.0.0 — 3-sigma outlier detection on numeric metric +// history. For each (entity, metric) key with at least MIN_POINTS numeric +// observations retained in state history, the latest point is compared to the +// baseline (all points excluding the latest): flag when +// |latest − mean(baseline)| > 3 · σ(baseline), with population σ. +// +// Direct statistical statement about an observed quantity → tier B. +// Confidence scales with the z-score: 0.6 at z = 3, +0.05 per unit of z, +// capped at 0.95 (an infinite z — zero-variance baseline — pins the cap). +import type { InferenceRecord, ObservationRecord } from "@pact-tailor/ontology"; +import { addSource, methodOf, type IntelRule, type RuleContext } from "./types.js"; + +const NAME = "anomaly-detection"; +const VERSION = "1.0.0"; +export const MIN_POINTS = 12; +const CONFIDENCE_CAP = 0.95; + +function confidenceFromZ(z: number): number { + if (!Number.isFinite(z)) return CONFIDENCE_CAP; + return Math.min(CONFIDENCE_CAP, 0.6 + 0.05 * (z - 3)); +} + +function numericHistory(ctx: RuleContext, entityId: string, metric: string): ObservationRecord[] { + return ctx.state.history(entityId, metric).filter((o) => typeof o.value === "number"); +} + +/** Scan one (entity, metric) key; returns zero or one anomaly record. */ +export function anomaliesForKey( + ctx: RuleContext, + entityId: string, + metric: string, + method: string = `rule:${NAME}@${VERSION}`, +): InferenceRecord[] { + const points = numericHistory(ctx, entityId, metric); + if (points.length < MIN_POINTS) return []; + const latest = points[points.length - 1] as ObservationRecord; + const baseline = points.slice(0, -1); + const values = baseline.map((o) => o.value as number); + const mean = values.reduce((a, b) => a + b, 0) / values.length; + const variance = values.reduce((a, b) => a + (b - mean) ** 2, 0) / values.length; + const sigma = Math.sqrt(variance); + const deviation = Math.abs((latest.value as number) - mean); + + // Flag strictly past 3σ. Zero-variance baseline: any deviation is anomalous. + if (sigma === 0 ? deviation === 0 : deviation <= 3 * sigma) return []; + const z = sigma === 0 ? Number.POSITIVE_INFINITY : deviation / sigma; + + // Evidence: the anomalous observation plus the two most recent baseline points. + const b1 = baseline[baseline.length - 1] as ObservationRecord; + const b2 = baseline[baseline.length - 2] as ObservationRecord; + const evidence: InferenceRecord["evidence"] = [ + { kind: "observation", ref: latest.observation_id, role: "supports" }, + { kind: "observation", ref: b1.observation_id, role: "supports" }, + { kind: "observation", ref: b2.observation_id, role: "supports" }, + ]; + const sources: string[] = []; + addSource(sources, latest.source_id); + addSource(sources, b1.source_id); + + const first = baseline[0] as ObservationRecord; + return [ + { + inference_id: ctx.mintId(NAME), + claim: `anomalous ${metric} on ${entityId}: latest ${latest.value} vs baseline mean ${mean.toFixed(2)} ± ${sigma.toFixed(2)} (z=${Number.isFinite(z) ? z.toFixed(2) : "inf"})`, + claim_structured: { + subject: entityId, + predicate: "exhibits_anomaly", + object: metric, + qualifiers: { + latest: latest.value, + baseline_mean: mean, + baseline_sigma: sigma, + z_score: Number.isFinite(z) ? z : null, + baseline_points: baseline.length, + }, + }, + tier: "B", + confidence: confidenceFromZ(z), + status: "INFERRED", + method, + evidence, + sources, + produced_at: ctx.now(), + event_time_range: { from: first.event_time, to: latest.event_time }, + }, + ]; +} + +export const anomalyDetection: IntelRule = { + name: NAME, + version: VERSION, + run(ctx: RuleContext): InferenceRecord[] { + const out: InferenceRecord[] = []; + for (const state of ctx.state.projection.all()) { + out.push(...anomaliesForKey(ctx, state.entity_id, state.metric, methodOf(this))); + } + return out; + }, +}; diff --git a/intelligence/packages/intel/src/rules/cargo-inference.ts b/intelligence/packages/intel/src/rules/cargo-inference.ts new file mode 100644 index 0000000..e4a975e --- /dev/null +++ b/intelligence/packages/intel/src/rules/cargo-inference.ts @@ -0,0 +1,112 @@ +// cargo-inference@1.0.0 — infer the probable cargo of a vessel from where it +// loads. STRICT precondition: the vessel must have a LOADS_AT edge to a +// Terminal — vessels without one are NEVER speculated about. +// +// Commodity candidates for the terminal, in order of preference: +// - terminal.properties.commodities (declared string list), else +// - the `properties.commodity` of incoming TRANSPORTS edges. +// One inference per candidate commodity. A single-commodity terminal gives +// confidence 0.85; a multi-commodity terminal 0.6 per candidate. Multi-hop +// (vessel → terminal → commodity) → tier C. +import type { InferenceRecord, RelationshipRecord } from "@pact-tailor/ontology"; +import { addSource, methodOf, type IntelRule, type RuleContext } from "./types.js"; + +const NAME = "cargo-inference"; +const VERSION = "1.0.0"; + +interface Candidate { + commodity: string; + /** TRANSPORTS edge the commodity came from, when not declared on the terminal. */ + via: RelationshipRecord | null; +} + +function terminalCommodities(ctx: RuleContext, terminalId: string): Candidate[] { + const terminal = ctx.graph.node(terminalId); + if (!terminal) return []; + const props = (terminal.properties ?? {}) as Record; + const declared = props["commodities"]; + const out: Candidate[] = []; + const seen = new Set(); + if (Array.isArray(declared)) { + for (const c of declared) { + if (typeof c === "string" && c.length > 0 && !seen.has(c)) { + seen.add(c); + out.push({ commodity: c, via: null }); + } + } + } + if (out.length === 0) { + for (const step of ctx.graph.neighbors(terminalId, { direction: "in", relTypes: ["TRANSPORTS"] })) { + const commodity = (step.rel.properties ?? {})["commodity"]; + if (typeof commodity === "string" && commodity.length > 0 && !seen.has(commodity)) { + seen.add(commodity); + out.push({ commodity, via: step.rel }); + } + } + } + return out.sort((a, b) => a.commodity.localeCompare(b.commodity)); +} + +export const cargoInference: IntelRule = { + name: NAME, + version: VERSION, + run(ctx: RuleContext): InferenceRecord[] { + const out: InferenceRecord[] = []; + for (const vessel of ctx.graph.allNodes()) { + if (vessel.entity_type !== "Vessel") continue; + // Only LOADS_AT edges that point at a Terminal count. No edge → no claim. + const loadEdges = ctx.graph + .neighbors(vessel.entity_id, { direction: "out", relTypes: ["LOADS_AT"] }) + .filter((step) => ctx.graph.node(step.to)?.entity_type === "Terminal"); + if (loadEdges.length === 0) continue; + + const positionState = ctx.state.current(vessel.entity_id, "vessel.position")[0]; + + for (const step of loadEdges) { + const terminal = ctx.graph.node(step.to); + if (!terminal) continue; + const candidates = terminalCommodities(ctx, step.to); + if (candidates.length === 0) continue; // nothing known about the terminal's cargo + const confidence = candidates.length === 1 ? 0.85 : 0.6; + + for (const candidate of candidates) { + const evidence: InferenceRecord["evidence"] = [ + { kind: "structural", ref: step.rel.rel_id, role: "supports" }, + ]; + if (candidate.via) evidence.push({ kind: "structural", ref: candidate.via.rel_id, role: "supports" }); + if (positionState !== undefined) { + evidence.push({ kind: "observation", ref: positionState.observation_id, role: "supports" }); + } + + const sources: string[] = []; + addSource(sources, step.rel.source); + if (candidate.via) addSource(sources, candidate.via.source); + if (positionState !== undefined) addSource(sources, positionState.source_id); + + out.push({ + inference_id: ctx.mintId(NAME), + claim: `${vessel.name} probably carries ${candidate.commodity} (loads at ${terminal.name})`, + claim_structured: { + subject: vessel.entity_id, + predicate: "carries", + object: candidate.commodity, + qualifiers: { + terminal: terminal.entity_id, + candidate_count: candidates.length, + }, + }, + tier: "C", + confidence, + status: "INFERRED", + method: methodOf(this), + evidence, + contrary_evidence: [], + sources, + produced_at: ctx.now(), + }); + } + } + } + return out; + }, +}; diff --git a/intelligence/packages/intel/src/rules/gap-synthesis.ts b/intelligence/packages/intel/src/rules/gap-synthesis.ts new file mode 100644 index 0000000..b4fce93 --- /dev/null +++ b/intelligence/packages/intel/src/rules/gap-synthesis.ts @@ -0,0 +1,56 @@ +// gap-synthesis@1.0.0 — find entities that SHOULD have an observability-gap +// record but don't: observability ESTIMATED or NOT_OBSERVABLE, no gap record +// in the loaded registry, and an entity type we know the keystone metric for. +// The emitted inferences feed the gap-registry review loop (a human or a +// downstream process turns accepted ones into real gap records). +// +// Multi-hop editorial claim about our own observability posture → tier C. +import type { EntityType, InferenceRecord } from "@pact-tailor/ontology"; +import { methodOf, type IntelRule, type RuleContext } from "./types.js"; + +const NAME = "gap-synthesis"; +const VERSION = "1.0.0"; + +/** Entity types with a well-known keystone live metric. Others are skipped. */ +export const SUGGESTED_METRIC: Partial> = { + Smelter: "power.load.mw", + Terminal: "production.rate.tph", + Mine: "production.rate.tph", + LNGPlant: "production.rate.tph", + DataCentre: "power.load.mw", +}; + +export const gapSynthesis: IntelRule = { + name: NAME, + version: VERSION, + run(ctx: RuleContext): InferenceRecord[] { + const covered = new Set(ctx.gaps.map((g) => g.entity_id)); + const out: InferenceRecord[] = []; + for (const entity of ctx.graph.allNodes()) { + if (entity.observability !== "ESTIMATED" && entity.observability !== "NOT_OBSERVABLE") continue; + if (covered.has(entity.entity_id)) continue; + const metric = SUGGESTED_METRIC[entity.entity_type]; + if (metric === undefined) continue; + if (entity.sources.length === 0) continue; // cannot cite a source — skip honestly + + out.push({ + inference_id: ctx.mintId(NAME), + claim: `observability gap: ${entity.name} lacks live ${metric}`, + claim_structured: { + subject: entity.entity_id, + predicate: "lacks_live_metric", + object: metric, + qualifiers: { observability: entity.observability, entity_type: entity.entity_type }, + }, + tier: "C", + confidence: 0.9, + status: "INFERRED", + method: methodOf(this), + evidence: [{ kind: "structural", ref: entity.entity_id, role: "supports" }], + sources: [...entity.sources], + produced_at: ctx.now(), + }); + } + return out; + }, +}; diff --git a/intelligence/packages/intel/src/rules/index.ts b/intelligence/packages/intel/src/rules/index.ts new file mode 100644 index 0000000..328c2cd --- /dev/null +++ b/intelligence/packages/intel/src/rules/index.ts @@ -0,0 +1,19 @@ +export { type IntelRule, type RuleContext, methodOf } from "./types.js"; +export { smelterUtilisation } from "./smelter-utilisation.js"; +export { cargoInference } from "./cargo-inference.js"; +export { anomalyDetection, anomaliesForKey, MIN_POINTS } from "./anomaly-detection.js"; +export { gapSynthesis, SUGGESTED_METRIC } from "./gap-synthesis.js"; + +import type { IntelRule } from "./types.js"; +import { smelterUtilisation } from "./smelter-utilisation.js"; +import { cargoInference } from "./cargo-inference.js"; +import { anomalyDetection } from "./anomaly-detection.js"; +import { gapSynthesis } from "./gap-synthesis.js"; + +/** All built-in rules, in the order the engine runs them. */ +export const ALL_RULES: readonly IntelRule[] = [ + smelterUtilisation, + cargoInference, + anomalyDetection, + gapSynthesis, +]; diff --git a/intelligence/packages/intel/src/rules/smelter-utilisation.ts b/intelligence/packages/intel/src/rules/smelter-utilisation.ts new file mode 100644 index 0000000..9aa05f5 --- /dev/null +++ b/intelligence/packages/intel/src/rules/smelter-utilisation.ts @@ -0,0 +1,118 @@ +// smelter-utilisation@1.0.0 — estimate the live electrical load of smelters +// whose state we cannot observe directly (observability ESTIMATED), using the +// grid region's live demand as the activity proxy and the smelter's declared +// typical load as the magnitude. +// +// Region resolution heuristic (documented, simplest-first): +// 1. If the smelter's grid_id points directly at a GridRegion node, use it. +// 2. Else, if a Generator SUPPLIES the smelter and that generator's grid_id +// points at a GridRegion, use that region. +// 3. Else, among GridRegion nodes whose grid_id equals the smelter's grid_id +// and which hold live grid.demand.mw state, use the first by entity_id. +// +// The emitted estimate is the smelter's properties.typical_load_mw (declared +// structural knowledge); the region demand observation is the evidence that +// the region is live and the proxy through which the estimate is derived. +// Single-hop derivation from a strong proxy → tier B. +import type { EntityRecord, InferenceRecord } from "@pact-tailor/ontology"; +import type { RelationshipRecord } from "@pact-tailor/ontology"; +import { addSource, methodOf, type IntelRule, type RuleContext } from "./types.js"; + +const NAME = "smelter-utilisation"; +const VERSION = "1.0.0"; + +/** Confidence: base 0.6, +0.15 when a structural SUPPLIES edge ties the smelter to its supply chain. */ +function score(hasSupplyEdge: boolean): number { + return hasSupplyEdge ? 0.75 : 0.6; +} + +function resolveRegion(ctx: RuleContext, smelter: EntityRecord): EntityRecord | null { + // 1. grid_id is itself a region. + if (smelter.grid_id) { + const direct = ctx.graph.node(smelter.grid_id); + if (direct && direct.entity_type === "GridRegion") return direct; + } + // 2. via a Generator that SUPPLIES the smelter. + for (const step of ctx.graph.neighbors(smelter.entity_id, { direction: "in", relTypes: ["SUPPLIES"] })) { + const generator = ctx.graph.node(step.from); + if (generator?.entity_type !== "Generator" || !generator.grid_id) continue; + const region = ctx.graph.node(generator.grid_id); + if (region && region.entity_type === "GridRegion") return region; + } + // 3. first GridRegion child of the smelter's grid that has live demand state. + if (smelter.grid_id) { + const candidates = ctx.graph + .allNodes() + .filter( + (n) => + n.entity_type === "GridRegion" && + n.grid_id === smelter.grid_id && + ctx.state.current(n.entity_id, "grid.demand.mw").length > 0, + ) + .sort((a, b) => a.entity_id.localeCompare(b.entity_id)); + if (candidates[0] !== undefined) return candidates[0]; + } + return null; +} + +function suppliesEdge(ctx: RuleContext, smelterId: string): RelationshipRecord | null { + for (const step of ctx.graph.neighbors(smelterId, { direction: "in", relTypes: ["SUPPLIES"] })) { + if (ctx.graph.node(step.from)?.entity_type === "Generator") return step.rel; + } + return null; +} + +export const smelterUtilisation: IntelRule = { + name: NAME, + version: VERSION, + run(ctx: RuleContext): InferenceRecord[] { + const out: InferenceRecord[] = []; + for (const entity of ctx.graph.allNodes()) { + if (entity.entity_type !== "Smelter" || entity.observability !== "ESTIMATED") continue; + + const props = (entity.properties ?? {}) as Record; + const typicalLoad = props["typical_load_mw"]; + if (typeof typicalLoad !== "number" || !Number.isFinite(typicalLoad)) continue; // no declared share — skip + + const region = resolveRegion(ctx, entity); + if (!region) continue; + const demandState = ctx.state.current(region.entity_id, "grid.demand.mw")[0]; + if (demandState === undefined) continue; // no live proxy — nothing honest to say + + const edge = suppliesEdge(ctx, entity.entity_id); + const evidence: InferenceRecord["evidence"] = [ + { kind: "observation", ref: demandState.observation_id, role: "supports" }, + ]; + if (edge) evidence.push({ kind: "structural", ref: edge.rel_id, role: "supports" }); + + const sources: string[] = []; + addSource(sources, demandState.source_id); + if (edge) addSource(sources, edge.source); + + out.push({ + inference_id: ctx.mintId(NAME), + claim: `${entity.name}: estimated live load ≈ ${typicalLoad} MW (declared typical load; ${region.name} demand live at ${demandState.value} MW)`, + claim_structured: { + subject: entity.entity_id, + predicate: "has_estimated_load_mw", + object: String(typicalLoad), + qualifiers: { + unit: "MW", + proxy_region: region.entity_id, + proxy_metric: "grid.demand.mw", + proxy_value: demandState.value, + }, + }, + tier: "B", + confidence: score(edge !== null), + status: "INFERRED", + method: methodOf(this), + evidence, + sources, + produced_at: ctx.now(), + event_time_range: { from: demandState.event_time, to: demandState.event_time }, + }); + } + return out; + }, +}; diff --git a/intelligence/packages/intel/src/rules/types.ts b/intelligence/packages/intel/src/rules/types.ts new file mode 100644 index 0000000..6a4cd98 --- /dev/null +++ b/intelligence/packages/intel/src/rules/types.ts @@ -0,0 +1,48 @@ +// Rule contract for @pact-tailor/intel. A rule is a pure, deterministic, +// versioned function over the knowledge graph + real-time state. It emits +// evidence-linked InferenceRecords (class C) and NOTHING else — no state +// mutation, no graph mutation, no I/O. +// +// Honesty invariants (enforced by the engine's schema validation and by +// convention here): +// - `method` is always `rule:${name}@${version}` — reproducible provenance. +// - Every evidence ref points at a REAL id: an observation_id held in state +// history, a rel_id / entity_id present in the graph. Rules never mint +// fresh ULIDs to use as evidence (`newUlidRefs?: never` documents this — +// there is no facility in the context for making up references). +import type { InferenceRecord, ObservabilityGapRecord } from "@pact-tailor/ontology"; +import type { KnowledgeGraph } from "@pact-tailor/graph"; +import type { StateEngine } from "@pact-tailor/state"; + +export interface RuleContext { + graph: KnowledgeGraph; + state: StateEngine; + /** Observability-gap registry records, loaded once at boot (read-only). */ + gaps: readonly ObservabilityGapRecord[]; + /** The rule run's single timestamp — every record in one run shares it. */ + now(): string; + /** + * Mint the next inference_id for this run: + * `infer:${now().slice(0, 10)}:${slug}-${n}` with an engine-scoped counter, + * so ids stay unique across repeated runs on the same day. + */ + mintId(slug: string): string; + /** Rules must not fabricate evidence refs — see module header. */ + newUlidRefs?: never; +} + +export interface IntelRule { + name: string; + version: string; + run(ctx: RuleContext): InferenceRecord[]; +} + +/** `rule:${name}@${version}` — the method string for a rule's records. */ +export function methodOf(rule: Pick): string { + return `rule:${rule.name}@${rule.version}`; +} + +/** Deduplicating push helper for source id lists (order-preserving). */ +export function addSource(sources: string[], id: string | undefined): void { + if (id !== undefined && id.length > 0 && !sources.includes(id)) sources.push(id); +} diff --git a/intelligence/packages/intel/test/engine.test.mjs b/intelligence/packages/intel/test/engine.test.mjs new file mode 100644 index 0000000..6f19dcf --- /dev/null +++ b/intelligence/packages/intel/test/engine.test.mjs @@ -0,0 +1,136 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { KnowledgeGraph } from "@pact-tailor/graph"; +import { StateEngine } from "@pact-tailor/state"; +import { MedallionStore } from "@pact-tailor/store"; +import { ulid } from "@pact-tailor/ontology"; +import { IntelEngine, ALL_RULES, MIN_POINTS } from "../dist/index.js"; + +function entity(id, type, over = {}) { + return { + entity_id: id, + entity_type: type, + name: `Name of ${id}`, + data_class: "structural", + observability: "KNOWN_LIVE", + sources: ["source:test:feed"], + ...over, + }; +} + +function obs(entityId, metric, value, eventTime) { + return { + observation_id: ulid(Date.parse(eventTime)), + entity_id: entityId, + metric, + value, + event_time: eventTime, + ingest_time: eventTime, + source_id: "source:test:feed", + feed_id: "feed:test:main", + quality: "good", + }; +} + +function engineFixture() { + const graph = new KnowledgeGraph(); + graph.load( + [ + entity("term:test:open", "Terminal", { observability: "ESTIMATED" }), + entity("mine:test:dark", "Mine", { observability: "NOT_OBSERVABLE" }), + ], + [], + ); + const state = new StateEngine(); + return { graph, state }; +} + +test("runAll runs every rule, byEntity filters on subject, ids stay unique across runs", () => { + const { graph, state } = engineFixture(); + const engine = new IntelEngine({ graph, state, gaps: [] }); + assert.equal(ALL_RULES.length, 4); + + const first = engine.runAll("2026-08-12T01:00:00Z"); + assert.equal(first.length, 2); // two gap-synthesis records; other rules have no targets + const second = engine.runAll("2026-08-12T02:00:00Z"); + assert.equal(second.length, 2); + + const ids = engine.all().map((r) => r.inference_id); + assert.equal(new Set(ids).size, ids.length, "inference ids must be unique across runs"); + assert.equal(engine.all().length, 4); + + const forMine = engine.byEntity("mine:test:dark"); + assert.equal(forMine.length, 2); + assert.ok(forMine.every((r) => r.claim_structured.subject === "mine:test:dark")); + assert.equal(engine.byEntity("mine:test:unknown").length, 0); +}); + +test("runRule rejects unknown rule names", () => { + const { graph, state } = engineFixture(); + const engine = new IntelEngine({ graph, state, gaps: [] }); + assert.throws(() => engine.runRule("no-such-rule"), /unknown rule/); +}); + +test("persistTo appends only new records to the gold inference log", () => { + const { graph, state } = engineFixture(); + const engine = new IntelEngine({ graph, state, gaps: [] }); + const varDir = mkdtempSync(join(tmpdir(), "pt-intel-")); + const store = new MedallionStore(varDir); + + engine.runAll("2026-08-12T01:00:00Z"); + assert.equal(engine.persistTo(store), 2); + assert.equal(store.gold.readInferences().length, 2); + // Idempotent when nothing new was produced. + assert.equal(engine.persistTo(store), 0); + assert.equal(store.gold.readInferences().length, 2); + + engine.runAll("2026-08-12T02:00:00Z"); + assert.equal(engine.persistTo(store), 2); + assert.equal(store.gold.readInferences().length, 4); +}); + +test("attach() re-runs anomaly-detection for the affected key only (debounced)", async () => { + const graph = new KnowledgeGraph(); + graph.load([entity("region:test:north", "GridRegion")], []); + const state = new StateEngine(); + const baseline = []; + for (let i = 0; i < MIN_POINTS; i++) { + baseline.push( + obs( + "region:test:north", + "grid.demand.mw", + i % 2 === 0 ? 99 : 101, + new Date(Date.UTC(2026, 7, 12, 0, i * 5)).toISOString(), + ), + ); + } + state.ingest(baseline); + + const engine = new IntelEngine({ graph, state, gaps: [], reactDebounceMs: 20 }); + const seen = []; + const unsubscribe = engine.onInference((r) => seen.push(r)); + const detach = engine.attach(); + + // A non-anomalous delta must not produce an inference after debounce. + state.ingest([obs("region:test:north", "grid.demand.mw", 100, "2026-08-12T02:00:00Z")]); + await new Promise((resolve) => setTimeout(resolve, 80)); + assert.equal(seen.length, 0); + + // An outlier delta triggers a scoped anomaly record. + state.ingest([obs("region:test:north", "grid.demand.mw", 500, "2026-08-12T03:00:00Z")]); + await new Promise((resolve) => setTimeout(resolve, 80)); + assert.equal(seen.length, 1); + assert.equal(seen[0].claim_structured.subject, "region:test:north"); + assert.equal(seen[0].claim_structured.object, "grid.demand.mw"); + assert.equal(seen[0].method, "rule:anomaly-detection@1.0.0"); + + detach(); + unsubscribe(); + // After detach, further deltas are ignored. + state.ingest([obs("region:test:north", "grid.demand.mw", 900, "2026-08-12T04:00:00Z")]); + await new Promise((resolve) => setTimeout(resolve, 80)); + assert.equal(engine.all().length, 1); +}); diff --git a/intelligence/packages/intel/test/rules.test.mjs b/intelligence/packages/intel/test/rules.test.mjs new file mode 100644 index 0000000..5c56f6f --- /dev/null +++ b/intelligence/packages/intel/test/rules.test.mjs @@ -0,0 +1,299 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { KnowledgeGraph } from "@pact-tailor/graph"; +import { StateEngine } from "@pact-tailor/state"; +import { createValidators, ulid } from "@pact-tailor/ontology"; +import { IntelEngine, MIN_POINTS } from "../dist/index.js"; + +const validators = createValidators(); + +function entity(id, type, over = {}) { + return { + entity_id: id, + entity_type: type, + name: `Name of ${id}`, + data_class: "structural", + observability: "KNOWN_LIVE", + sources: ["source:test:feed"], + ...over, + }; +} + +function rel(id, type, from, to, over = {}) { + return { + rel_id: id, + rel_type: type, + from_id: from, + to_id: to, + confidence: 1, + source: "source:test:feed", + method: "declared", + ...over, + }; +} + +function obs(entityId, metric, value, eventTime, over = {}) { + return { + observation_id: ulid(Date.parse(eventTime)), + entity_id: entityId, + metric, + value, + event_time: eventTime, + ingest_time: eventTime, + source_id: "source:test:feed", + feed_id: "feed:test:main", + quality: "good", + ...over, + }; +} + +function assertValid(record) { + const result = validators.inference(record); + assert.ok(result.ok, `inference must validate: ${result.ok ? "" : result.errors.join("; ")}`); +} + +// ── smelter-utilisation ──────────────────────────────────────────────────── + +function smelterFixture() { + const graph = new KnowledgeGraph(); + graph.load( + [ + entity("grid:test", "Grid"), + entity("region:test:north", "GridRegion", { grid_id: "grid:test" }), + entity("gen:test:alpha", "Generator", { grid_id: "region:test:north" }), + entity("smelter:test:alpha", "Smelter", { + observability: "ESTIMATED", + grid_id: "grid:test", + properties: { typical_load_mw: 800 }, + }), + ], + [rel("rel:alpha-supplies-smelter", "SUPPLIES", "gen:test:alpha", "smelter:test:alpha")], + ); + const state = new StateEngine(); + const demand = obs("region:test:north", "grid.demand.mw", 5000, "2026-08-12T00:00:00Z"); + state.ingest([demand]); + return { graph, state, demand }; +} + +test("smelter-utilisation emits an evidence-linked tier-B estimate", () => { + const { graph, state, demand } = smelterFixture(); + const engine = new IntelEngine({ graph, state, gaps: [] }); + const records = engine.runRule("smelter-utilisation", "2026-08-12T01:00:00Z"); + assert.equal(records.length, 1); + const [record] = records; + assertValid(record); + assert.match(record.claim, /estimated live load ≈ 800 MW/); + assert.equal(record.claim_structured.subject, "smelter:test:alpha"); + assert.equal(record.tier, "B"); + assert.equal(record.status, "INFERRED"); + assert.equal(record.method, "rule:smelter-utilisation@1.0.0"); + assert.equal(record.confidence, 0.75); + assert.ok(record.evidence.length >= 2); + // Evidence refs are REAL: the demand observation and the SUPPLIES edge. + const refs = record.evidence.map((e) => e.ref); + assert.ok(refs.includes(demand.observation_id)); + assert.ok(refs.includes("rel:alpha-supplies-smelter")); + assert.match(record.inference_id, /^infer:2026-08-12:smelter-utilisation-1$/); +}); + +test("smelter-utilisation skips smelters without typical_load_mw or without ESTIMATED observability", () => { + const graph = new KnowledgeGraph(); + graph.load( + [ + entity("grid:test", "Grid"), + entity("region:test:north", "GridRegion", { grid_id: "grid:test" }), + // no typical_load_mw: + entity("smelter:test:bare", "Smelter", { observability: "ESTIMATED", grid_id: "grid:test" }), + // not ESTIMATED: + entity("smelter:test:live", "Smelter", { + observability: "KNOWN_LIVE", + grid_id: "grid:test", + properties: { typical_load_mw: 500 }, + }), + ], + [], + ); + const state = new StateEngine(); + state.ingest([obs("region:test:north", "grid.demand.mw", 5000, "2026-08-12T00:00:00Z")]); + const engine = new IntelEngine({ graph, state, gaps: [] }); + assert.equal(engine.runRule("smelter-utilisation").length, 0); +}); + +test("smelter-utilisation emits nothing without a live demand proxy", () => { + const { graph } = smelterFixture(); + const engine = new IntelEngine({ graph, state: new StateEngine(), gaps: [] }); + assert.equal(engine.runRule("smelter-utilisation").length, 0); +}); + +// ── cargo-inference ──────────────────────────────────────────────────────── + +function cargoFixture({ commodities = ["coal"], withPosition = true } = {}) { + const graph = new KnowledgeGraph(); + graph.load( + [ + entity("term:test:coal", "Terminal", { properties: { commodities } }), + entity("vessel:test:bulk1", "Vessel"), + entity("vessel:test:idle", "Vessel"), + ], + [rel("rel:bulk1-loads-coal", "LOADS_AT", "vessel:test:bulk1", "term:test:coal")], + ); + const state = new StateEngine(); + let position = null; + if (withPosition) { + position = obs("vessel:test:bulk1", "vessel.position", { lat: -23.8, lon: 151.2 }, "2026-08-12T00:05:00Z"); + state.ingest([position]); + } + return { graph, state, position }; +} + +test("cargo-inference emits tier-C carries claim for LOADS_AT vessels", () => { + const { graph, state, position } = cargoFixture(); + const engine = new IntelEngine({ graph, state, gaps: [] }); + const records = engine.runRule("cargo-inference"); + assert.equal(records.length, 1); + const [record] = records; + assertValid(record); + assert.equal(record.tier, "C"); + assert.equal(record.confidence, 0.85); + assert.deepEqual(record.contrary_evidence, []); + assert.equal(record.claim_structured.subject, "vessel:test:bulk1"); + assert.equal(record.claim_structured.predicate, "carries"); + assert.equal(record.claim_structured.object, "coal"); + const refs = record.evidence.map((e) => e.ref); + assert.ok(refs.includes("rel:bulk1-loads-coal")); + assert.ok(refs.includes(position.observation_id)); +}); + +test("cargo-inference NEVER emits for vessels without a LOADS_AT edge", () => { + const { graph, state } = cargoFixture(); + const engine = new IntelEngine({ graph, state, gaps: [] }); + const records = engine.runRule("cargo-inference"); + assert.ok(records.every((r) => r.claim_structured.subject !== "vessel:test:idle")); +}); + +test("cargo-inference multi-commodity terminal drops confidence to 0.6 per candidate", () => { + const { graph, state } = cargoFixture({ commodities: ["coal", "alumina"], withPosition: false }); + const engine = new IntelEngine({ graph, state, gaps: [] }); + const records = engine.runRule("cargo-inference"); + assert.equal(records.length, 2); + for (const record of records) { + assertValid(record); + assert.equal(record.confidence, 0.6); + } + assert.deepEqual(records.map((r) => r.claim_structured.object).sort(), ["alumina", "coal"]); +}); + +test("cargo-inference falls back to incoming TRANSPORTS edges for commodity", () => { + const graph = new KnowledgeGraph(); + graph.load( + [ + entity("term:test:bare", "Terminal"), + entity("mine:test:black", "Mine"), + entity("vessel:test:bulk1", "Vessel"), + ], + [ + rel("rel:bulk1-loads-bare", "LOADS_AT", "vessel:test:bulk1", "term:test:bare"), + rel("rel:black-transports-bare", "TRANSPORTS", "mine:test:black", "term:test:bare", { + properties: { commodity: "coal" }, + }), + ], + ); + const engine = new IntelEngine({ graph, state: new StateEngine(), gaps: [] }); + const records = engine.runRule("cargo-inference"); + assert.equal(records.length, 1); + assert.equal(records[0].claim_structured.object, "coal"); + assert.ok(records[0].evidence.some((e) => e.ref === "rel:black-transports-bare")); +}); + +// ── anomaly-detection ────────────────────────────────────────────────────── + +// points defaults to MIN_POINTS + 1 so the baseline (points − 1) is even and the +// alternating 99/101 values give exactly mean 100, population sigma 1. +function anomalyFixture(latestValue, points = MIN_POINTS + 1) { + const graph = new KnowledgeGraph(); + graph.load([entity("region:test:north", "GridRegion")], []); + const state = new StateEngine(); + const baseline = []; + // Alternating 99/101 baseline: mean 100, population sigma 1. + for (let i = 0; i < points - 1; i++) { + baseline.push( + obs("region:test:north", "grid.demand.mw", i % 2 === 0 ? 99 : 101, new Date(Date.UTC(2026, 7, 12, 0, i * 5)).toISOString()), + ); + } + const latest = obs("region:test:north", "grid.demand.mw", latestValue, new Date(Date.UTC(2026, 7, 12, 12, 0)).toISOString()); + state.ingest([...baseline, latest]); + return { graph, state, latest, baseline }; +} + +test("anomaly-detection stays quiet within 3 sigma", () => { + const { graph, state } = anomalyFixture(103); // deviation == 3 sigma exactly — not strictly past + const engine = new IntelEngine({ graph, state, gaps: [] }); + assert.equal(engine.runRule("anomaly-detection").length, 0); +}); + +test("anomaly-detection flags strictly past 3 sigma with z-scaled confidence", () => { + const { graph, state, latest, baseline } = anomalyFixture(110); // z = 10 + const engine = new IntelEngine({ graph, state, gaps: [] }); + const records = engine.runRule("anomaly-detection"); + assert.equal(records.length, 1); + const [record] = records; + assertValid(record); + assert.match(record.claim, /^anomalous grid\.demand\.mw/); + assert.equal(record.tier, "B"); + assert.ok(record.confidence > 0.6 && record.confidence <= 0.95); + assert.equal(record.confidence, 0.95); // z=10 caps + assert.equal(record.evidence.length, 3); + const historyIds = new Set([latest, ...baseline].map((o) => o.observation_id)); + for (const e of record.evidence) { + assert.equal(e.kind, "observation"); + assert.ok(historyIds.has(e.ref), "evidence refs must be real observation ids"); + } + assert.equal(record.evidence[0].ref, latest.observation_id); +}); + +test("anomaly-detection requires at least 12 points", () => { + const { graph, state } = anomalyFixture(200, MIN_POINTS - 1); + const engine = new IntelEngine({ graph, state, gaps: [] }); + assert.equal(engine.runRule("anomaly-detection").length, 0); +}); + +// ── gap-synthesis ────────────────────────────────────────────────────────── + +test("gap-synthesis emits only for uncovered, mapped, unobservable entities", () => { + const graph = new KnowledgeGraph(); + graph.load( + [ + entity("smelter:test:covered", "Smelter", { observability: "ESTIMATED" }), + entity("term:test:open", "Terminal", { observability: "ESTIMATED" }), + entity("mine:test:dark", "Mine", { observability: "NOT_OBSERVABLE" }), + entity("gen:test:est", "Generator", { observability: "ESTIMATED" }), // no metric mapping + entity("smelter:test:live", "Smelter", { observability: "KNOWN_LIVE" }), + ], + [], + ); + const gaps = [ + { + gap_id: "gap:test:covered-load", + entity_id: "smelter:test:covered", + desired_metric: "power.load.mw", + commercial_value: "high", + strategic_value: "high", + priority: "P1", + status: "open", + }, + ]; + const engine = new IntelEngine({ graph, state: new StateEngine(), gaps }); + const records = engine.runRule("gap-synthesis"); + const subjects = records.map((r) => r.claim_structured.subject).sort(); + assert.deepEqual(subjects, ["mine:test:dark", "term:test:open"]); + for (const record of records) { + assertValid(record); + assert.equal(record.tier, "C"); + assert.equal(record.confidence, 0.9); + assert.equal(record.claim_structured.object, "production.rate.tph"); + assert.match(record.claim, /^observability gap: /); + assert.deepEqual(record.evidence, [ + { kind: "structural", ref: record.claim_structured.subject, role: "supports" }, + ]); + } +}); From 274017459ba73194510acc32cf684378e911fb23 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 02:05:38 +0000 Subject: [PATCH 14/16] feat(intelligence): REST + SSE API + SDUI composer (phase 18) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Zero-dep node:http server (reference-server style) composing registry ⊕ graph ⊕ state ⊕ gaps ⊕ inferences at read time: entities with bbox/type/ grid filters, grid live summaries, graph traversal endpoints, sources, gaps, inferences, SSE stream with per-connection filters and heartbeats, and server-driven UI: deterministic per-type panel templates + live situation rules (anomaly hoists inferences, no-live-data hoists the gap card), with composeWithAI as the documented identity stub — panels reference endpoints, never inline values. Honest replay mode drains remaining silver observations through the real ingest path on a compressed clock. 27 tests. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014yVGqBtvM1zNAd9tvUVNzm --- intelligence/packages/api/package.json | 6 +- intelligence/packages/api/src/app.ts | 228 +++++++++ intelligence/packages/api/src/cli.ts | 53 ++ intelligence/packages/api/src/index.ts | 22 +- .../packages/api/src/sdui/composer.ts | 130 +++++ .../api/src/sdui/templates/default.ts | 7 + .../api/src/sdui/templates/generator.ts | 12 + .../packages/api/src/sdui/templates/grid.ts | 15 + .../packages/api/src/sdui/templates/index.ts | 32 ++ .../packages/api/src/sdui/templates/mine.ts | 13 + .../packages/api/src/sdui/templates/port.ts | 12 + .../packages/api/src/sdui/templates/shared.ts | 34 ++ .../api/src/sdui/templates/smelter.ts | 14 + .../api/src/sdui/templates/terminal.ts | 13 + .../packages/api/src/sdui/templates/vessel.ts | 18 + intelligence/packages/api/src/server.ts | 379 ++++++++++++++ intelligence/packages/api/test/api.test.mjs | 482 ++++++++++++++++++ intelligence/packages/api/test/sdui.test.mjs | 195 +++++++ 18 files changed, 1662 insertions(+), 3 deletions(-) create mode 100644 intelligence/packages/api/src/app.ts create mode 100644 intelligence/packages/api/src/cli.ts create mode 100644 intelligence/packages/api/src/sdui/composer.ts create mode 100644 intelligence/packages/api/src/sdui/templates/default.ts create mode 100644 intelligence/packages/api/src/sdui/templates/generator.ts create mode 100644 intelligence/packages/api/src/sdui/templates/grid.ts create mode 100644 intelligence/packages/api/src/sdui/templates/index.ts create mode 100644 intelligence/packages/api/src/sdui/templates/mine.ts create mode 100644 intelligence/packages/api/src/sdui/templates/port.ts create mode 100644 intelligence/packages/api/src/sdui/templates/shared.ts create mode 100644 intelligence/packages/api/src/sdui/templates/smelter.ts create mode 100644 intelligence/packages/api/src/sdui/templates/terminal.ts create mode 100644 intelligence/packages/api/src/sdui/templates/vessel.ts create mode 100644 intelligence/packages/api/src/server.ts create mode 100644 intelligence/packages/api/test/api.test.mjs create mode 100644 intelligence/packages/api/test/sdui.test.mjs diff --git a/intelligence/packages/api/package.json b/intelligence/packages/api/package.json index 0c05044..297f7ac 100644 --- a/intelligence/packages/api/package.json +++ b/intelligence/packages/api/package.json @@ -9,11 +9,15 @@ "exports": { ".": "./dist/index.js" }, + "bin": { + "pt-api": "dist/cli.js" + }, "engines": { "node": ">=20" }, "scripts": { - "build": "tsc -p tsconfig.json" + "build": "tsc -p tsconfig.json", + "test": "npm run build && node --test \"test/*.test.mjs\"" }, "dependencies": { "@pact-tailor/graph": "*", diff --git a/intelligence/packages/api/src/app.ts b/intelligence/packages/api/src/app.ts new file mode 100644 index 0000000..aead870 --- /dev/null +++ b/intelligence/packages/api/src/app.ts @@ -0,0 +1,228 @@ +// buildApp — composes registry ⊕ graph ⊕ state ⊕ gaps ⊕ inferences into one +// application object the HTTP layer serves. Also owns the SSE hub (typed +// delta/inference broadcasting) and the replay clock. +import type { ServerResponse } from "node:http"; +import { + createValidators, + type InferenceRecord, + type ObservationRecord, + type Validators, +} from "@pact-tailor/ontology"; +import { loadRegistry, gridEntitiesFrom, checkIntegrity, type RegistryData } from "@pact-tailor/registry"; +import { KnowledgeGraph } from "@pact-tailor/graph"; +import { StateEngine, type StateDelta } from "@pact-tailor/state"; +import { MedallionStore } from "@pact-tailor/store"; +import { IntelEngine } from "@pact-tailor/intel"; + +// --------------------------------------------------------------------------- +// SSE hub +// --------------------------------------------------------------------------- + +export interface SseFilters { + /** Allow-list of entity_ids (null = all). */ + entities: Set | null; + /** Allow-list of metrics (null = all). */ + metrics: Set | null; +} + +interface SseClient extends SseFilters { + res: ServerResponse; + heartbeat: NodeJS.Timeout; +} + +export const SSE_HEARTBEAT_MS = 15_000; + +export class SseHub { + private readonly clients = new Set(); + + constructor(private readonly heartbeatMs: number = SSE_HEARTBEAT_MS) {} + + get size(): number { + return this.clients.size; + } + + /** Attach a response as an SSE stream: headers, hello event, heartbeat, cleanup. */ + subscribe(res: ServerResponse, filters: SseFilters, hello: unknown): void { + res.writeHead(200, { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }); + res.write(`event: hello\ndata: ${JSON.stringify(hello)}\n\n`); + const heartbeat = setInterval(() => { + res.write(": heartbeat\n\n"); + }, this.heartbeatMs); + heartbeat.unref?.(); + const client: SseClient = { res, heartbeat, ...filters }; + this.clients.add(client); + res.on("close", () => { + clearInterval(heartbeat); + this.clients.delete(client); + }); + } + + /** Forward a state delta as `event: state.updated | state.corrected`. */ + broadcastDelta(delta: StateDelta): void { + for (const client of this.clients) { + if (client.entities && !client.entities.has(delta.state.entity_id)) continue; + if (client.metrics && !client.metrics.has(delta.state.metric)) continue; + client.res.write( + `event: ${delta.type}\ndata: ${JSON.stringify({ + state: delta.state, + previous: delta.previous, + data_class: "telemetry", + })}\n\n`, + ); + } + } + + /** Forward a freshly produced inference as `event: inference`. */ + broadcastInference(inference: InferenceRecord): void { + for (const client of this.clients) { + if (client.entities && !client.entities.has(inference.claim_structured.subject)) continue; + client.res.write( + `event: inference\ndata: ${JSON.stringify({ inference, data_class: "derived" })}\n\n`, + ); + } + } + + closeAll(): void { + for (const client of this.clients) { + clearInterval(client.heartbeat); + client.res.end(); + } + this.clients.clear(); + } +} + +// --------------------------------------------------------------------------- +// App +// --------------------------------------------------------------------------- + +export interface BuildAppOptions { + /** Registry root (normally intelligence/data). */ + dataDir: string; + /** Medallion root (normally intelligence/var). */ + varDir: string; + /** Replay mode: boot on half the silver history, tick the rest through the real ingest path. */ + replay?: boolean; + /** Replay tick interval, ms (default 3000; tests use a smaller value). */ + replayTickMs?: number; +} + +export interface IntelApp { + registry: RegistryData; + graph: KnowledgeGraph; + state: StateEngine; + store: MedallionStore; + intel: IntelEngine; + validators: Validators; + sse: SseHub; + replay: boolean; + bootedAt: string; + /** Observations still queued for the replay ticker. */ + replayRemaining(): number; + startReplayTicker(): void; + stopReplayTicker(): void; + /** Detach all subscriptions, stop timers, end SSE streams. */ + close(): void; +} + +export function buildApp(opts: BuildAppOptions): IntelApp { + const validators = createValidators(); + + const { data, errors } = loadRegistry(opts.dataDir, validators); + if (errors.length > 0) { + throw new Error(`api: registry failed to load (${errors.length} error(s)):\n${errors.join("\n")}`); + } + const integrityProblems = checkIntegrity(data); + if (integrityProblems.length > 0) { + // Referential problems are surfaced, not fatal — schema-valid records still serve. + console.warn(`api: registry integrity problems (${integrityProblems.length}): ${integrityProblems.join("; ")}`); + } + + const graph = new KnowledgeGraph(); + graph.load([...data.entities, ...gridEntitiesFrom(data.grids)], data.relationships); + + const store = new MedallionStore(opts.varDir); + const state = new StateEngine(); + const replay = opts.replay === true; + + // Replay clock (honest by construction): no synthetic observations and no + // re-stamped event_times. All silver observations are sorted by event_time; + // the first half is ingested at boot, and the ticker ingests the remainder + // — one event-second batch per tick — through the REAL ingest path, so + // every delta consumers see is a genuine engine delta over genuine data on + // a compressed clock. When the queue drains the ticker stops (no looping). + let replayQueue: ObservationRecord[] = []; + if (replay) { + const all: ObservationRecord[] = []; + for (const feedId of store.silver.listFeeds()) all.push(...store.silver.readFeed(feedId)); + all.sort((a, b) => Date.parse(a.event_time) - Date.parse(b.event_time)); + const boot = Math.floor(all.length / 2); + state.ingest(all.slice(0, boot)); + replayQueue = all.slice(boot); + } else { + state.loadFromSilver(store); + } + + const intel = new IntelEngine({ graph, state, gaps: data.gaps }); + intel.runAll(); + intel.persistTo(store); + const detachIntel = intel.attach(); // react mode: scoped anomaly re-runs on deltas + + const sse = new SseHub(); + const offDelta = state.onDelta((delta) => sse.broadcastDelta(delta)); + const offInference = intel.onInference((inference) => sse.broadcastInference(inference)); + + let ticker: NodeJS.Timeout | null = null; + const tickMs = opts.replayTickMs ?? 3000; + + const stopReplayTicker = (): void => { + if (ticker !== null) { + clearInterval(ticker); + ticker = null; + } + }; + + const startReplayTicker = (): void => { + if (!replay || ticker !== null) return; + ticker = setInterval(() => { + if (replayQueue.length === 0) { + console.log("replay drained"); + stopReplayTicker(); + return; + } + const head = replayQueue[0] as ObservationRecord; + const second = head.event_time.slice(0, 19); + const batch: ObservationRecord[] = []; + while (replayQueue.length > 0 && (replayQueue[0] as ObservationRecord).event_time.slice(0, 19) === second) { + batch.push(replayQueue.shift() as ObservationRecord); + } + state.ingest(batch); + }, tickMs); + ticker.unref?.(); + }; + + return { + registry: data, + graph, + state, + store, + intel, + validators, + sse, + replay, + bootedAt: new Date().toISOString(), + replayRemaining: () => replayQueue.length, + startReplayTicker, + stopReplayTicker, + close(): void { + stopReplayTicker(); + offDelta(); + offInference(); + detachIntel(); + sse.closeAll(); + }, + }; +} diff --git a/intelligence/packages/api/src/cli.ts b/intelligence/packages/api/src/cli.ts new file mode 100644 index 0000000..6d9e8ef --- /dev/null +++ b/intelligence/packages/api/src/cli.ts @@ -0,0 +1,53 @@ +#!/usr/bin/env node +// pt-api — start the PACT.TAILOR intelligence API. +// pt-api [--port ] [--data ] [--var ] [--replay] +// Port default 4200 (env PT_API_PORT overrides); data/var default to the +// workspace's intelligence/data and intelligence/var directories. +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { buildApp } from "./app.js"; +import { startServer } from "./server.js"; + +interface CliArgs { + port: number; + dataDir: string; + varDir: string; + replay: boolean; +} + +function parseArgs(argv: string[]): CliArgs { + const here = path.dirname(fileURLToPath(import.meta.url)); // packages/api/dist + const workspaceRoot = path.resolve(here, "../../.."); // intelligence/ + const args: CliArgs = { + port: Number.parseInt(process.env["PT_API_PORT"] ?? "4200", 10), + dataDir: path.join(workspaceRoot, "data"), + varDir: path.join(workspaceRoot, "var"), + replay: false, + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--port") args.port = Number.parseInt(argv[++i] ?? "", 10); + else if (arg === "--data") args.dataDir = path.resolve(argv[++i] ?? ""); + else if (arg === "--var") args.varDir = path.resolve(argv[++i] ?? ""); + else if (arg === "--replay") args.replay = true; + else { + console.error(`pt-api: unknown argument ${arg}`); + process.exit(2); + } + } + if (!Number.isFinite(args.port) || args.port < 0) { + console.error(`pt-api: invalid port ${args.port}`); + process.exit(2); + } + return args; +} + +const args = parseArgs(process.argv.slice(2)); +const app = buildApp({ dataDir: args.dataDir, varDir: args.varDir, replay: args.replay }); +const server = startServer(app, args.port); + +for (const sig of ["SIGINT", "SIGTERM"] as const) { + process.on(sig, () => { + server.close(() => process.exit(0)); + }); +} diff --git a/intelligence/packages/api/src/index.ts b/intelligence/packages/api/src/index.ts index 60dcd6d..39b6b92 100644 --- a/intelligence/packages/api/src/index.ts +++ b/intelligence/packages/api/src/index.ts @@ -1,2 +1,20 @@ -// @pact-tailor/api — REST + SSE server + SDUI composer. Populated in the API commit. -export {}; +// @pact-tailor/api — REST + SSE server + SDUI composer. +export { + buildApp, + SseHub, + SSE_HEARTBEAT_MS, + type BuildAppOptions, + type IntelApp, + type SseFilters, +} from "./app.js"; +export { startServer, route } from "./server.js"; +export { + composePanelDoc, + computeSituations, + applySituationRules, + composeWithAI, + DEFAULT_FEED_CADENCE_MS, + STALENESS_MULTIPLIER, + type ComposeInputs, +} from "./sdui/composer.js"; +export { templateFor, type Template } from "./sdui/templates/index.js"; diff --git a/intelligence/packages/api/src/sdui/composer.ts b/intelligence/packages/api/src/sdui/composer.ts new file mode 100644 index 0000000..714b0c2 --- /dev/null +++ b/intelligence/packages/api/src/sdui/composer.ts @@ -0,0 +1,130 @@ +// SDUI composer (docs/SDUI.md, implemented exactly): +// 1. deterministic per-type base template, +// 2. situation computation from live state at composition time, +// 3. situation rules inject/reorder panels, +// 4. composeWithAI — the declared AI seam, currently the identity function. +// +// Panels reference endpoints only; the composer never inlines data values. +// The HTTP handler validates the finished document against +// createValidators().sduiPanel and fails loud (500) on violation. +import type { + CurrentMetricState, +} from "@pact-tailor/state"; +import type { + EntityRecord, + InferenceRecord, + ObservabilityGapRecord, + SduiPanel, + SduiPanelDoc, + SduiSituation, +} from "@pact-tailor/ontology"; +import { gapCard, inferenceList, templateFor } from "./templates/index.js"; + +/** + * Default assumed feed cadence when no telemetry-feed record is available: + * 30 minutes. Telemetry is "stale" when staleness exceeds 3× the cadence. + */ +export const DEFAULT_FEED_CADENCE_MS = 30 * 60 * 1000; +export const STALENESS_MULTIPLIER = 3; + +export interface ComposeInputs { + entity: EntityRecord; + /** Current per-metric states for the entity ([] = no live data). */ + states: CurrentMetricState[]; + /** Gap registry records targeting the entity. */ + gaps: ObservabilityGapRecord[]; + /** Inference records whose subject is the entity. */ + inferences: InferenceRecord[]; + /** Composition instant (ISO). Defaults to wall clock. */ + now?: string; +} + +/** Situation rules, computed from live state at composition time. */ +export function computeSituations(inputs: ComposeInputs): SduiSituation[] { + const nowMs = Date.parse(inputs.now ?? new Date().toISOString()); + const situations: SduiSituation[] = []; + + if (inputs.states.length === 0) situations.push("no-live-data"); + + const staleThreshold = STALENESS_MULTIPLIER * DEFAULT_FEED_CADENCE_MS; + const anyStale = inputs.states.some((s) => nowMs - Date.parse(s.event_time) > staleThreshold); + if (anyStale) situations.push("stale-telemetry"); + + if (inputs.gaps.length > 0) situations.push("has-gaps"); + + if (inputs.entity.observability === "ESTIMATED") situations.push("estimated-load"); + + const anomalyActive = inputs.inferences.some( + (i) => + i.method.startsWith("rule:anomaly-detection@") && + (i.status === "INFERRED" || i.status === "CORROBORATED"), + ); + if (anomalyActive) situations.push("anomaly-active"); + + return situations.length === 0 ? ["normal"] : situations; +} + +/** Move the first panel of the given kind to the front (inject if absent). */ +function hoist(layout: SduiPanel[], kind: SduiPanel["panel"], make: () => SduiPanel): SduiPanel[] { + const idx = layout.findIndex((p) => p.panel === kind); + const panel = idx >= 0 ? (layout[idx] as SduiPanel) : make(); + const rest = idx >= 0 ? [...layout.slice(0, idx), ...layout.slice(idx + 1)] : [...layout]; + return [panel, ...rest]; +} + +/** + * Situation rules inject or reorder panels: + * - has-gaps injects a gap-card when the template lacks one; + * - no-live-data hoists the gap-card to the top; + * - anomaly-active hoists the inference-list to the very top (applied last, + * so it wins over the gap-card hoist). + */ +export function applySituationRules( + layout: SduiPanel[], + situations: SduiSituation[], + entityId: string, +): SduiPanel[] { + let out = [...layout]; + if (situations.includes("has-gaps") && !out.some((p) => p.panel === "gap-card")) { + out.push(gapCard(entityId)); + } + if (situations.includes("no-live-data")) { + out = hoist(out, "gap-card", () => gapCard(entityId)); + } + if (situations.includes("anomaly-active")) { + out = hoist(out, "inference-list", () => inferenceList(entityId)); + } + return out; +} + +/** + * AI hook (docs/SDUI.md "AI hook") — the declared seam for model-driven + * composition. Contract: AI may reorder, group, annotate, or drop panels and + * add panels referencing declared endpoints; it may NOT inline data values, + * invent endpoints, or alter existing panels' data blocks. Output is + * schema-validated before serving, and violations fall back to the + * deterministic layout. The current implementation is the identity function — + * deterministic output ships unchanged — so the AI layer can land later + * without renegotiating this contract. + */ +export function composeWithAI( + _entity: EntityRecord, + _situations: SduiSituation[], + deterministicLayout: SduiPanel[], +): SduiPanel[] { + return deterministicLayout; +} + +/** Compose the full panel document for an entity. */ +export function composePanelDoc(inputs: ComposeInputs): SduiPanelDoc { + const situations = computeSituations(inputs); + const base = templateFor(inputs.entity.entity_type)(inputs.entity.entity_id); + const deterministic = applySituationRules(base, situations, inputs.entity.entity_id); + const layout = composeWithAI(inputs.entity, situations, deterministic); + return { + entity_id: inputs.entity.entity_id, + entity_type: inputs.entity.entity_type, + situation: situations, + layout, + }; +} diff --git a/intelligence/packages/api/src/sdui/templates/default.ts b/intelligence/packages/api/src/sdui/templates/default.ts new file mode 100644 index 0000000..10ef699 --- /dev/null +++ b/intelligence/packages/api/src/sdui/templates/default.ts @@ -0,0 +1,7 @@ +// Default template — for entity types without a bespoke layout. +import type { SduiPanel } from "@pact-tailor/ontology"; +import { graphNeighborhood, headlineState, inferenceList } from "./shared.js"; + +export function defaultTemplate(id: string): SduiPanel[] { + return [headlineState(id), graphNeighborhood(id), inferenceList(id)]; +} diff --git a/intelligence/packages/api/src/sdui/templates/generator.ts b/intelligence/packages/api/src/sdui/templates/generator.ts new file mode 100644 index 0000000..1cb29a6 --- /dev/null +++ b/intelligence/packages/api/src/sdui/templates/generator.ts @@ -0,0 +1,12 @@ +// Generator template — current MW output and the upstream fuel chain. +import type { SduiPanel } from "@pact-tailor/ontology"; +import { graphNeighborhood, headlineState, inferenceList, timeseries } from "./shared.js"; + +export function generatorTemplate(id: string): SduiPanel[] { + return [ + headlineState(id), + timeseries(id, "power.output.mw"), + graphNeighborhood(id, { depth: 2, direction: "upstream" }), + inferenceList(id), + ]; +} diff --git a/intelligence/packages/api/src/sdui/templates/grid.ts b/intelligence/packages/api/src/sdui/templates/grid.ts new file mode 100644 index 0000000..cebc2fa --- /dev/null +++ b/intelligence/packages/api/src/sdui/templates/grid.ts @@ -0,0 +1,15 @@ +// Grid / GridRegion template — leads with load, generation, price and flows. +import type { SduiPanel } from "@pact-tailor/ontology"; +import { graphNeighborhood, headlineState, inferenceList, timeseries } from "./shared.js"; + +export function gridTemplate(id: string): SduiPanel[] { + return [ + headlineState(id), + timeseries(id, "grid.demand.mw"), + { panel: "price-strip", data: { endpoint: `/api/intel/entities/${id}/observations?metric=market.price.energy` } }, + { panel: "fuel-mix", data: { endpoint: `/api/intel/entities/${id}/observations?metric=grid.generation.fuel_mix` } }, + { panel: "flow-arcs", data: { endpoint: `/api/intel/grids/${id}/summary` } }, + graphNeighborhood(id), + inferenceList(id), + ]; +} diff --git a/intelligence/packages/api/src/sdui/templates/index.ts b/intelligence/packages/api/src/sdui/templates/index.ts new file mode 100644 index 0000000..6aab5e6 --- /dev/null +++ b/intelligence/packages/api/src/sdui/templates/index.ts @@ -0,0 +1,32 @@ +// Deterministic per-type base templates (docs/SDUI.md "Composition" step 1). +import type { EntityType, SduiPanel } from "@pact-tailor/ontology"; +import { gridTemplate } from "./grid.js"; +import { generatorTemplate } from "./generator.js"; +import { smelterTemplate } from "./smelter.js"; +import { vesselTemplate } from "./vessel.js"; +import { portTemplate } from "./port.js"; +import { terminalTemplate } from "./terminal.js"; +import { mineTemplate } from "./mine.js"; +import { defaultTemplate } from "./default.js"; + +export { gridTemplate, generatorTemplate, smelterTemplate, vesselTemplate, portTemplate, terminalTemplate, mineTemplate, defaultTemplate }; +export { gapCard, graphNeighborhood, headlineState, inferenceList, timeseries } from "./shared.js"; + +export type Template = (entityId: string) => SduiPanel[]; + +const TEMPLATES: Partial> = { + Grid: gridTemplate, + GridRegion: gridTemplate, + Generator: generatorTemplate, + Smelter: smelterTemplate, + Vessel: vesselTemplate, + Port: portTemplate, + Terminal: terminalTemplate, + Berth: terminalTemplate, + Mine: mineTemplate, + CoalMine: mineTemplate, +}; + +export function templateFor(entityType: EntityType): Template { + return TEMPLATES[entityType] ?? defaultTemplate; +} diff --git a/intelligence/packages/api/src/sdui/templates/mine.ts b/intelligence/packages/api/src/sdui/templates/mine.ts new file mode 100644 index 0000000..7227073 --- /dev/null +++ b/intelligence/packages/api/src/sdui/templates/mine.ts @@ -0,0 +1,13 @@ +// Mine template — production rate, gaps, downstream transport chain, inferences. +import type { SduiPanel } from "@pact-tailor/ontology"; +import { gapCard, graphNeighborhood, headlineState, inferenceList, timeseries } from "./shared.js"; + +export function mineTemplate(id: string): SduiPanel[] { + return [ + headlineState(id), + timeseries(id, "production.rate.tph"), + gapCard(id), + graphNeighborhood(id, { depth: 2, direction: "downstream" }), + inferenceList(id), + ]; +} diff --git a/intelligence/packages/api/src/sdui/templates/port.ts b/intelligence/packages/api/src/sdui/templates/port.ts new file mode 100644 index 0000000..7f728e5 --- /dev/null +++ b/intelligence/packages/api/src/sdui/templates/port.ts @@ -0,0 +1,12 @@ +// Port template — headline, downstream trade neighborhood, gaps, inferences. +import type { SduiPanel } from "@pact-tailor/ontology"; +import { gapCard, graphNeighborhood, headlineState, inferenceList } from "./shared.js"; + +export function portTemplate(id: string): SduiPanel[] { + return [ + headlineState(id), + graphNeighborhood(id, { depth: 2 }), + gapCard(id), + inferenceList(id), + ]; +} diff --git a/intelligence/packages/api/src/sdui/templates/shared.ts b/intelligence/packages/api/src/sdui/templates/shared.ts new file mode 100644 index 0000000..a683144 --- /dev/null +++ b/intelligence/packages/api/src/sdui/templates/shared.ts @@ -0,0 +1,34 @@ +// Shared panel builders for the SDUI type templates. Panels reference +// ENDPOINTS only — never inline data values — so what the client renders is +// always the API's provenance-tagged truth at fetch time. +import type { SduiPanel } from "@pact-tailor/ontology"; + +export function headlineState(id: string): SduiPanel { + return { panel: "headline-state", data: { endpoint: `/api/intel/entities/${id}/state` } }; +} + +export function timeseries(id: string, metric: string, window = "24h"): SduiPanel { + return { + panel: "timeseries", + props: { metric, window }, + data: { endpoint: `/api/intel/entities/${id}/observations?metric=${metric}` }, + }; +} + +export function gapCard(id: string): SduiPanel { + return { panel: "gap-card", data: { endpoint: `/api/intel/gaps?entity=${id}` } }; +} + +export function graphNeighborhood(id: string, props: Record = { depth: 2 }): SduiPanel { + const depth = typeof props["depth"] === "number" ? props["depth"] : 2; + const direction = typeof props["direction"] === "string" ? `&direction=${props["direction"]}` : ""; + return { + panel: "graph-neighborhood", + props, + data: { endpoint: `/api/intel/entities/${id}/graph?depth=${depth}${direction}` }, + }; +} + +export function inferenceList(id: string): SduiPanel { + return { panel: "inference-list", data: { endpoint: `/api/intel/inferences?entity=${id}` } }; +} diff --git a/intelligence/packages/api/src/sdui/templates/smelter.ts b/intelligence/packages/api/src/sdui/templates/smelter.ts new file mode 100644 index 0000000..907bd8c --- /dev/null +++ b/intelligence/packages/api/src/sdui/templates/smelter.ts @@ -0,0 +1,14 @@ +// Smelter template — mirrors the worked example in docs/SDUI.md: headline, +// estimated-load timeseries, gap card, neighborhood, inferences. +import type { SduiPanel } from "@pact-tailor/ontology"; +import { gapCard, graphNeighborhood, headlineState, inferenceList, timeseries } from "./shared.js"; + +export function smelterTemplate(id: string): SduiPanel[] { + return [ + headlineState(id), + timeseries(id, "power.load.estimated_mw"), + gapCard(id), + graphNeighborhood(id), + inferenceList(id), + ]; +} diff --git a/intelligence/packages/api/src/sdui/templates/terminal.ts b/intelligence/packages/api/src/sdui/templates/terminal.ts new file mode 100644 index 0000000..e2ee539 --- /dev/null +++ b/intelligence/packages/api/src/sdui/templates/terminal.ts @@ -0,0 +1,13 @@ +// Terminal template — throughput, gaps, upstream commodity chain, inferences. +import type { SduiPanel } from "@pact-tailor/ontology"; +import { gapCard, graphNeighborhood, headlineState, inferenceList, timeseries } from "./shared.js"; + +export function terminalTemplate(id: string): SduiPanel[] { + return [ + headlineState(id), + timeseries(id, "production.rate.tph"), + gapCard(id), + graphNeighborhood(id, { depth: 2, direction: "upstream" }), + inferenceList(id), + ]; +} diff --git a/intelligence/packages/api/src/sdui/templates/vessel.ts b/intelligence/packages/api/src/sdui/templates/vessel.ts new file mode 100644 index 0000000..370b5e0 --- /dev/null +++ b/intelligence/packages/api/src/sdui/templates/vessel.ts @@ -0,0 +1,18 @@ +// Vessel template — position/speed headline, track, cargo status. Reported +// cargo (observations) and inferred cargo (inference-list) are separate +// panels so the two are visibly distinct in the client. +import type { SduiPanel } from "@pact-tailor/ontology"; +import { graphNeighborhood, headlineState, inferenceList } from "./shared.js"; + +export function vesselTemplate(id: string): SduiPanel[] { + return [ + headlineState(id), + { + panel: "vessel-track", + props: { metric: "vessel.position", window: "48h" }, + data: { endpoint: `/api/intel/entities/${id}/observations?metric=vessel.position` }, + }, + graphNeighborhood(id, { depth: 2 }), + inferenceList(id), + ]; +} diff --git a/intelligence/packages/api/src/server.ts b/intelligence/packages/api/src/server.ts new file mode 100644 index 0000000..b71023a --- /dev/null +++ b/intelligence/packages/api/src/server.ts @@ -0,0 +1,379 @@ +// Zero-dependency node:http REST + SSE server in the reference-server style: +// router on path segments, sendJson, 404 JSON envelope. Every entity-bearing +// response is tagged with data_class provenance so structural vs telemetry vs +// derived is always distinguishable. +import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import type { CurrentMetricState } from "@pact-tailor/state"; +import type { EntityRecord, RelType } from "@pact-tailor/ontology"; +import type { Path } from "@pact-tailor/graph"; +import type { IntelApp } from "./app.js"; +import { composePanelDoc } from "./sdui/composer.js"; + +// ─── helpers ────────────────────────────────────────────────────────────── + +function sendJson(res: ServerResponse, status: number, body: unknown): void { + const payload = JSON.stringify(body); + res.writeHead(status, { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(payload), + }); + res.end(payload); +} + +function notFound(res: ServerResponse): void { + sendJson(res, 404, { error: "not_found" }); +} + +function badRequest(res: ServerResponse, message: string): void { + sendJson(res, 400, { error: "bad_request", message }); +} + +function numericState(app: IntelApp, entityId: string, metric: string): CurrentMetricState | null { + const state = app.state.current(entityId, metric)[0]; + return state !== undefined && typeof state.value === "number" ? state : null; +} + +function csvSet(value: string | null): Set | null { + if (value === null || value.trim() === "") return null; + const items = value.split(",").map((s) => s.trim()).filter((s) => s.length > 0); + return items.length > 0 ? new Set(items) : null; +} + +function clampInt(raw: string | null, fallback: number, min: number, max: number): number | null { + if (raw === null || raw === "") return fallback; + const n = Number.parseInt(raw, 10); + if (!Number.isFinite(n) || String(n) !== raw.trim()) return null; + return Math.min(max, Math.max(min, n)); +} + +// ─── handlers ───────────────────────────────────────────────────────────── + +function handleGridSummary(app: IntelApp, gridId: string, res: ServerResponse): void { + const grid = app.registry.grids.find((g) => g.grid_id === gridId); + if (!grid) return notFound(res); + + const regionNodes = app.graph + .allNodes() + .filter((n) => n.entity_type === "GridRegion" && n.grid_id === gridId) + .sort((a, b) => a.entity_id.localeCompare(b.entity_id)); + + const demandStates = regionNodes + .map((r) => numericState(app, r.entity_id, "grid.demand.mw")) + .filter((s): s is CurrentMetricState => s !== null); + const generationStates = regionNodes + .map((r) => numericState(app, r.entity_id, "grid.generation.mw")) + .filter((s): s is CurrentMetricState => s !== null); + const priceStates = regionNodes + .map((r) => numericState(app, r.entity_id, "market.price.energy")) + .filter((s): s is CurrentMetricState => s !== null); + + const demand_mw = + demandStates.length === 0 + ? null + : { + value: demandStates.reduce((a, s) => a + (s.value as number), 0), + event_time: demandStates.map((s) => s.event_time).sort().at(-1) as string, + }; + const generation_mw = + generationStates.length === 0 + ? null + : { value: generationStates.reduce((a, s) => a + (s.value as number), 0) }; + // Price is not aggregated across regions — only reported when unambiguous + // (exactly one region has a live price). Per-region prices sit in regions_live. + const priceState = priceStates.length === 1 ? (priceStates[0] as CurrentMetricState) : null; + const price = priceState === null ? null : { value: priceState.value, event_time: priceState.event_time }; + + const flows = app.graph + .allNodes() + .filter((n) => n.entity_type === "Interconnector" && n.grid_id === gridId) + .sort((a, b) => a.entity_id.localeCompare(b.entity_id)) + .flatMap((icon) => { + const state = numericState(app, icon.entity_id, "intercon.flow.mw"); + if (state === null) return []; + return [{ entity_id: icon.entity_id, name: icon.name, mw: state.value, event_time: state.event_time }]; + }); + + const regions_live = regionNodes.map((r) => ({ + entity_id: r.entity_id, + demand_mw: numericState(app, r.entity_id, "grid.demand.mw")?.value ?? null, + price: numericState(app, r.entity_id, "market.price.energy")?.value ?? null, + generation_mw: numericState(app, r.entity_id, "grid.generation.mw")?.value ?? null, + })); + + sendJson(res, 200, { + grid, + regions: regionNodes.map((r) => ({ entity_id: r.entity_id, name: r.name })), + live: { demand_mw, price, generation_mw, flows }, + regions_live, + data_class: { grid: "structural", regions: "structural", live: "telemetry", regions_live: "telemetry" }, + }); +} + +function handleEntities(app: IntelApp, query: URLSearchParams, res: ServerResponse): void { + let entities = app.graph.allNodes(); + + const type = query.get("type"); + if (type !== null && type !== "") entities = entities.filter((e) => e.entity_type === type); + + const grid = query.get("grid"); + if (grid !== null && grid !== "") entities = entities.filter((e) => e.grid_id === grid); + + const observability = query.get("observability"); + if (observability !== null && observability !== "") { + entities = entities.filter((e) => e.observability === observability); + } + + const q = query.get("q"); + if (q !== null && q !== "") { + const needle = q.toLowerCase(); + entities = entities.filter( + (e) => + e.name.toLowerCase().includes(needle) || + (e.aliases ?? []).some((a) => a.toLowerCase().includes(needle)), + ); + } + + const bbox = query.get("bbox"); + if (bbox !== null && bbox !== "") { + const parts = bbox.split(",").map((p) => Number.parseFloat(p)); + if (parts.length !== 4 || parts.some((n) => !Number.isFinite(n))) { + return badRequest(res, "bbox must be minLon,minLat,maxLon,maxLat"); + } + const [minLon, minLat, maxLon, maxLat] = parts as [number, number, number, number]; + entities = entities.filter((e) => { + if (!e.geometry || e.geometry.type !== "Point") return false; + const lon = e.geometry.coordinates[0]; + const lat = e.geometry.coordinates[1]; + if (typeof lon !== "number" || typeof lat !== "number") return false; + return lon >= minLon && lon <= maxLon && lat >= minLat && lat <= maxLat; + }); + } + + entities = [...entities].sort((a, b) => a.entity_id.localeCompare(b.entity_id)); + sendJson(res, 200, { entities, data_class: "structural" }); +} + +function handleEntityDetail(app: IntelApp, id: string, res: ServerResponse): void { + const entity = app.graph.node(id); + if (!entity) return notFound(res); + const sources = entity.sources.map( + (sid) => app.registry.sources.find((s) => s.source_id === sid) ?? sid, + ); + sendJson(res, 200, { + entity, + provenance: { data_class: "structural", sources }, + state: app.state.current(id), + gaps: app.registry.gaps.filter((g) => g.entity_id === id), + inference_count: app.intel.byEntity(id).length, + data_class: { entity: "structural", state: "telemetry", gaps: "structural", inferences: "derived" }, + }); +} + +function handleEntityGraph(app: IntelApp, id: string, query: URLSearchParams, res: ServerResponse): void { + if (!app.graph.node(id)) return notFound(res); + const direction = query.get("direction") ?? "both"; + if (direction !== "upstream" && direction !== "downstream" && direction !== "both") { + return badRequest(res, "direction must be upstream|downstream|both"); + } + const depth = clampInt(query.get("depth"), 2, 1, 5); + if (depth === null) return badRequest(res, "depth must be an integer"); + const relsParam = csvSet(query.get("rels")); + const relTypes = relsParam !== null ? ([...relsParam] as RelType[]) : undefined; + + const directions: Array<"upstream" | "downstream"> = + direction === "both" ? ["upstream", "downstream"] : [direction]; + const paths: Path[] = []; + for (const dir of directions) { + paths.push( + ...app.graph.traverse({ + start: id, + direction: dir, + maxDepth: depth, + ...(relTypes !== undefined ? { relTypes } : {}), + }), + ); + } + + const nodeIds = new Set([id]); + for (const path of paths) for (const nodeId of path.nodes) nodeIds.add(nodeId); + const nodes = [...nodeIds] + .sort() + .map((nid) => app.graph.node(nid)) + .filter((n): n is EntityRecord => n !== null); + + sendJson(res, 200, { paths, nodes, data_class: "structural" }); +} + +function handleGraphPath(app: IntelApp, query: URLSearchParams, res: ServerResponse): void { + const from = query.get("from"); + const to = query.get("to"); + if (!from || !to) return badRequest(res, "from and to are required"); + if (!app.graph.node(from)) return notFound(res); + const maxDepth = clampInt(query.get("maxDepth"), 4, 1, 6); + if (maxDepth === null) return badRequest(res, "maxDepth must be an integer"); + const paths = app.graph + .traverse({ start: from, direction: "downstream", maxDepth }) + .filter((p) => p.nodes[p.nodes.length - 1] === to); + sendJson(res, 200, { from, to, paths, data_class: "structural" }); +} + +function handleSdui(app: IntelApp, id: string, res: ServerResponse): void { + const entity = app.graph.node(id); + if (!entity) return notFound(res); + const doc = composePanelDoc({ + entity, + states: app.state.current(id), + gaps: app.registry.gaps.filter((g) => g.entity_id === id), + inferences: app.intel.byEntity(id), + }); + // Fail loud: a composed document that violates the SDUI schema is a bug. + const result = app.validators.sduiPanel(doc); + if (!result.ok) { + sendJson(res, 500, { error: "sdui_invalid", details: result.errors }); + return; + } + sendJson(res, 200, doc); +} + +function metaBody(app: IntelApp): Record { + return { + counts: { + entities: app.graph.nodeCount(), + relationships: app.graph.edgeCount(), + sources: app.registry.sources.length, + gaps: app.registry.gaps.length, + grids: app.registry.grids.length, + inferences: app.intel.all().length, + }, + replay: app.replay, + generated_at: new Date().toISOString(), + }; +} + +function handleStream(app: IntelApp, query: URLSearchParams, res: ServerResponse): void { + app.sse.subscribe( + res, + { entities: csvSet(query.get("entities")), metrics: csvSet(query.get("metrics")) }, + { service: "pact-tailor-intel-api", ...metaBody(app) }, + ); +} + +// ─── router ─────────────────────────────────────────────────────────────── + +export function route(app: IntelApp, req: IncomingMessage, res: ServerResponse): void { + const url = new URL(req.url ?? "/", "http://localhost"); + const segs = url.pathname.split("/").filter(Boolean).map(decodeURIComponent); + const query = url.searchParams; + const method = req.method ?? "GET"; + + if (method !== "GET") return notFound(res); + + if (segs.length === 1 && segs[0] === "healthz") { + return sendJson(res, 200, { status: "ok", service: "pact-tailor-intel-api" }); + } + + if (segs[0] !== "api" || segs[1] !== "intel") return notFound(res); + const rest = segs.slice(2); + const [head, id, sub] = rest; + + if (head === "grids") { + if (rest.length === 1) return sendJson(res, 200, { grids: app.registry.grids, data_class: "structural" }); + if (rest.length === 3 && sub === "summary" && id !== undefined) return handleGridSummary(app, id, res); + return notFound(res); + } + + if (head === "entities") { + if (rest.length === 1) return handleEntities(app, query, res); + if (id === undefined) return notFound(res); + if (rest.length === 2) return handleEntityDetail(app, id, res); + if (rest.length === 3 && sub === "state") { + if (!app.graph.node(id)) return notFound(res); + return sendJson(res, 200, { entity_id: id, states: app.state.current(id), data_class: "telemetry" }); + } + if (rest.length === 3 && sub === "observations") { + if (!app.graph.node(id)) return notFound(res); + const metric = query.get("metric"); + if (metric === null || metric === "") return badRequest(res, "metric query parameter is required"); + const from = query.get("from"); + const to = query.get("to"); + const observations = app.state.history(id, metric, { + ...(from !== null && from !== "" ? { from } : {}), + ...(to !== null && to !== "" ? { to } : {}), + }); + return sendJson(res, 200, { entity_id: id, metric, observations, data_class: "telemetry" }); + } + if (rest.length === 3 && sub === "graph") return handleEntityGraph(app, id, query, res); + return notFound(res); + } + + if (head === "graph" && rest.length === 2 && id === "path") return handleGraphPath(app, query, res); + + if (head === "sources") { + if (rest.length === 1) return sendJson(res, 200, { sources: app.registry.sources, data_class: "structural" }); + if (rest.length === 2 && id !== undefined) { + const source = app.registry.sources.find((s) => s.source_id === id); + return source ? sendJson(res, 200, { source, data_class: "structural" }) : notFound(res); + } + return notFound(res); + } + + if (head === "gaps" && rest.length === 1) { + let gaps = app.registry.gaps; + const entity = query.get("entity"); + if (entity !== null && entity !== "") gaps = gaps.filter((g) => g.entity_id === entity); + const priority = query.get("priority"); + if (priority !== null && priority !== "") gaps = gaps.filter((g) => g.priority === priority); + const status = query.get("status"); + if (status !== null && status !== "") gaps = gaps.filter((g) => g.status === status); + return sendJson(res, 200, { gaps, data_class: "structural" }); + } + + if (head === "inferences") { + if (rest.length === 1) { + const entity = query.get("entity"); + let inferences = entity !== null && entity !== "" ? app.intel.byEntity(entity) : app.intel.all(); + const status = query.get("status"); + if (status !== null && status !== "") inferences = inferences.filter((i) => i.status === status); + return sendJson(res, 200, { inferences, data_class: "derived" }); + } + if (rest.length === 2 && id !== undefined) { + const inference = app.intel.byId(id); + return inference ? sendJson(res, 200, { inference, data_class: "derived" }) : notFound(res); + } + return notFound(res); + } + + if (head === "sdui" && rest.length === 2 && id !== undefined) return handleSdui(app, id, res); + + if (head === "meta" && rest.length === 1) return sendJson(res, 200, metaBody(app)); + + if (head === "stream" && rest.length === 1) return handleStream(app, query, res); + + return notFound(res); +} + +// ─── bootstrap ──────────────────────────────────────────────────────────── + +/** + * Start the HTTP server on 127.0.0.1:port (0 = ephemeral). Prints + * `listening on http://127.0.0.1:{port}` once bound, and starts the replay + * ticker when the app was built in replay mode. Closing the server closes + * the app (ticker, subscriptions, SSE streams). + */ +export function startServer(app: IntelApp, port: number): Server { + const server = createServer((req, res) => { + try { + route(app, req, res); + } catch (err) { + sendJson(res, 500, { error: "internal", message: (err as Error).message }); + } + }); + server.listen(port, "127.0.0.1", () => { + const addr = server.address(); + const actual = addr !== null && typeof addr === "object" ? addr.port : port; + console.log(`listening on http://127.0.0.1:${actual}`); + if (app.replay) app.startReplayTicker(); + }); + server.on("close", () => app.close()); + return server; +} diff --git a/intelligence/packages/api/test/api.test.mjs b/intelligence/packages/api/test/api.test.mjs new file mode 100644 index 0000000..852700f --- /dev/null +++ b/intelligence/packages/api/test/api.test.mjs @@ -0,0 +1,482 @@ +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { MedallionStore } from "@pact-tailor/store"; +import { ulid } from "@pact-tailor/ontology"; +import { buildApp, startServer } from "../dist/index.js"; + +// ── fixture registry (YAML) ──────────────────────────────────────────────── + +function writeRegistry(root) { + mkdirSync(join(root, "registries", "grids"), { recursive: true }); + mkdirSync(join(root, "registries", "sources"), { recursive: true }); + mkdirSync(join(root, "registries", "gaps"), { recursive: true }); + mkdirSync(join(root, "entities", "test"), { recursive: true }); + mkdirSync(join(root, "relationships", "test"), { recursive: true }); + + writeFileSync( + join(root, "registries", "sources", "test.yaml"), + `records: + - source_id: source:test:feed + publisher: Test Publisher + dataset: Test Dataset + api_type: rest + url: https://example.com/api + coverage: { grids: [grid:test], countries: [au] } + data_class: live + licensing: { class: open } + verification_status: documented_only + last_checked: "2026-08-12" +`, + ); + writeFileSync( + join(root, "registries", "grids", "test.yaml"), + `records: + - grid_id: grid:test + name: Test Grid + kind: isolated_grid + countries: [au] + frequency_hz: 50 + live_data: { availability: rich, source_ids: [source:test:feed] } + sources: [source:test:feed] +`, + ); + writeFileSync( + join(root, "registries", "gaps", "test.yaml"), + `records: + - gap_id: gap:test:smelter-load + entity_id: smelter:test:alpha + desired_metric: power.load.mw + commercial_value: high + strategic_value: critical + priority: P1 + status: open +`, + ); + writeFileSync( + join(root, "entities", "test", "entities.yaml"), + `records: + - entity_id: region:test:north + entity_type: GridRegion + name: North Region + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [151.0, -23.0] } + country: au + grid_id: grid:test + sources: [source:test:feed] + - entity_id: region:test:south + entity_type: GridRegion + name: South Region + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [147.0, -32.0] } + country: au + grid_id: grid:test + sources: [source:test:feed] + - entity_id: gen:test:alpha + entity_type: Generator + name: Alpha Power Station + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [151.2, -23.5] } + country: au + grid_id: grid:test + properties: { fuel: coal, capacity_mw: 1600 } + sources: [source:test:feed] + - entity_id: smelter:test:alpha + entity_type: Smelter + name: Alpha Aluminium Smelter + aliases: [Alpha Smelter] + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [151.3, -23.9] } + country: au + grid_id: grid:test + properties: { product: aluminium, typical_load_mw: 800 } + sources: [source:test:feed] + - entity_id: term:test:coal + entity_type: Terminal + name: Coal Export Terminal + data_class: structural + observability: ESTIMATED + geometry: { type: Point, coordinates: [151.25, -23.85] } + country: au + grid_id: grid:test + properties: { commodities: [coal] } + sources: [source:test:feed] + - entity_id: vessel:test:bulk1 + entity_type: Vessel + name: Bulk Carrier One + data_class: structural + observability: KNOWN_LIVE + geometry: { type: Point, coordinates: [152.5, -23.0] } + properties: { vessel_type: bulk-carrier, synthetic: true } + sources: [source:test:feed] + - entity_id: intercon:test:link + entity_type: Interconnector + name: Test Link + data_class: structural + observability: KNOWN_LIVE + country: au + grid_id: grid:test + sources: [source:test:feed] +`, + ); + writeFileSync( + join(root, "relationships", "test", "rels.yaml"), + `records: + - rel_id: rel:alpha-supplies-smelter + rel_type: SUPPLIES + from_id: gen:test:alpha + to_id: smelter:test:alpha + confidence: 1.0 + source: source:test:feed + method: declared + - rel_id: rel:bulk1-loads-coal + rel_type: LOADS_AT + from_id: vessel:test:bulk1 + to_id: term:test:coal + confidence: 1.0 + source: source:test:feed + method: declared +`, + ); +} + +// ── fixture silver observations ──────────────────────────────────────────── + +function obs(entityId, metric, value, eventTime, feed = "feed:test:main") { + return { + observation_id: ulid(Date.parse(eventTime)), + entity_id: entityId, + metric, + value, + event_time: eventTime, + ingest_time: eventTime, + source_id: "source:test:feed", + feed_id: feed, + quality: "good", + }; +} + +function writeSilver(varDir) { + const store = new MedallionStore(varDir); + const observations = [ + obs("region:test:north", "grid.demand.mw", 5000, "2026-08-12T00:00:00Z"), + obs("region:test:north", "grid.demand.mw", 5100, "2026-08-12T00:05:00Z"), + obs("region:test:south", "grid.demand.mw", 3000, "2026-08-12T00:10:00Z"), + obs("region:test:north", "market.price.energy", 88.5, "2026-08-12T00:15:00Z"), + obs("region:test:north", "grid.generation.mw", 4500, "2026-08-12T00:20:00Z"), + obs("intercon:test:link", "intercon.flow.mw", 300, "2026-08-12T00:25:00Z"), + obs("vessel:test:bulk1", "vessel.position", { lat: -23.8, lon: 151.9 }, "2026-08-12T00:30:00Z", "feed:test:ais"), + ]; + store.silver.append(observations); + return observations; +} + +// ── boot ─────────────────────────────────────────────────────────────────── + +const dataDir = mkdtempSync(join(tmpdir(), "pt-api-data-")); +const varDir = mkdtempSync(join(tmpdir(), "pt-api-var-")); +writeRegistry(dataDir); +writeSilver(varDir); + +let app; +let server; +let base; + +function listen(theApp) { + return new Promise((resolve) => { + const srv = startServer(theApp, 0); + srv.on("listening", () => resolve(srv)); + }); +} + +async function get(path) { + const res = await fetch(`${base}${path}`); + const body = await res.json(); + return { status: res.status, body }; +} + +before(async () => { + app = buildApp({ dataDir, varDir, replay: false }); + server = await listen(app); + base = `http://127.0.0.1:${server.address().port}`; +}); + +after(async () => { + server.closeAllConnections?.(); + await new Promise((resolve) => server.close(resolve)); +}); + +// ── endpoint tests ───────────────────────────────────────────────────────── + +test("GET /healthz", async () => { + const { status, body } = await get("/healthz"); + assert.equal(status, 200); + assert.deepEqual(body, { status: "ok", service: "pact-tailor-intel-api" }); +}); + +test("GET /api/intel/grids", async () => { + const { status, body } = await get("/api/intel/grids"); + assert.equal(status, 200); + assert.equal(body.grids.length, 1); + assert.equal(body.grids[0].grid_id, "grid:test"); +}); + +test("GET /api/intel/grids/{id}/summary aggregates region state", async () => { + const { status, body } = await get("/api/intel/grids/grid:test/summary"); + assert.equal(status, 200); + assert.equal(body.grid.grid_id, "grid:test"); + assert.deepEqual( + body.regions.map((r) => r.entity_id), + ["region:test:north", "region:test:south"], + ); + assert.equal(body.live.demand_mw.value, 8100); // 5100 (latest north) + 3000 (south) + assert.equal(body.live.demand_mw.event_time, "2026-08-12T00:10:00Z"); + assert.equal(body.live.generation_mw.value, 4500); + assert.equal(body.live.price.value, 88.5); // unambiguous: only north has price + assert.equal(body.live.flows.length, 1); + assert.equal(body.live.flows[0].entity_id, "intercon:test:link"); + assert.equal(body.live.flows[0].mw, 300); + const north = body.regions_live.find((r) => r.entity_id === "region:test:north"); + assert.deepEqual(north, { entity_id: "region:test:north", demand_mw: 5100, price: 88.5, generation_mw: 4500 }); + assert.equal(body.data_class.live, "telemetry"); +}); + +test("GET /api/intel/grids/{id}/summary 404s on unknown grid", async () => { + assert.equal((await get("/api/intel/grids/grid:nope/summary")).status, 404); +}); + +test("GET /api/intel/entities with filters", async () => { + assert.equal((await get("/api/intel/entities")).body.entities.length, 8); // 7 + materialised grid node + + const smelters = (await get("/api/intel/entities?type=Smelter")).body.entities; + assert.equal(smelters.length, 1); + assert.equal(smelters[0].entity_id, "smelter:test:alpha"); + + assert.equal((await get("/api/intel/entities?grid=grid:test")).body.entities.length, 6); + assert.equal((await get("/api/intel/entities?observability=ESTIMATED")).body.entities.length, 2); + + const q = (await get("/api/intel/entities?q=alpha")).body.entities; + assert.deepEqual(q.map((e) => e.entity_id).sort(), ["gen:test:alpha", "smelter:test:alpha"]); + + const inBox = (await get("/api/intel/entities?bbox=150,-25,152,-23")).body.entities; + assert.deepEqual( + inBox.map((e) => e.entity_id).sort(), + ["gen:test:alpha", "region:test:north", "smelter:test:alpha", "term:test:coal"], + ); + + assert.equal((await get("/api/intel/entities?bbox=150,-25,152")).status, 400); +}); + +test("GET /api/intel/entities/{id} composes provenance, state, gaps, inferences", async () => { + const { status, body } = await get("/api/intel/entities/smelter:test:alpha"); + assert.equal(status, 200); + assert.equal(body.entity.entity_id, "smelter:test:alpha"); + assert.equal(body.provenance.data_class, "structural"); + assert.equal(body.provenance.sources[0].source_id, "source:test:feed"); // resolved to the full record + assert.deepEqual(body.state, []); // no live telemetry for the smelter itself + assert.equal(body.gaps.length, 1); + assert.equal(body.inference_count, 1); // smelter-utilisation estimate + assert.deepEqual(body.data_class, { + entity: "structural", + state: "telemetry", + gaps: "structural", + inferences: "derived", + }); + + assert.equal((await get("/api/intel/entities/smelter:test:nope")).status, 404); +}); + +test("GET /api/intel/entities/{id}/state", async () => { + const { status, body } = await get("/api/intel/entities/region:test:north/state"); + assert.equal(status, 200); + assert.equal(body.entity_id, "region:test:north"); + assert.deepEqual( + body.states.map((s) => s.metric), + ["grid.demand.mw", "grid.generation.mw", "market.price.energy"], + ); + assert.equal(body.data_class, "telemetry"); +}); + +test("GET /api/intel/entities/{id}/observations requires metric and honours range", async () => { + assert.equal((await get("/api/intel/entities/region:test:north/observations")).status, 400); + + const all = await get("/api/intel/entities/region:test:north/observations?metric=grid.demand.mw"); + assert.equal(all.status, 200); + assert.equal(all.body.observations.length, 2); + + const ranged = await get( + "/api/intel/entities/region:test:north/observations?metric=grid.demand.mw&from=2026-08-12T00:03:00Z", + ); + assert.equal(ranged.body.observations.length, 1); + assert.equal(ranged.body.observations[0].value, 5100); +}); + +test("GET /api/intel/entities/{id}/graph traverses with filters", async () => { + const { status, body } = await get("/api/intel/entities/vessel:test:bulk1/graph?direction=downstream&depth=1"); + assert.equal(status, 200); + assert.equal(body.paths.length, 1); + assert.deepEqual(body.paths[0].nodes, ["vessel:test:bulk1", "term:test:coal"]); + assert.ok(body.nodes.some((n) => n.entity_id === "term:test:coal")); + + const filtered = await get("/api/intel/entities/vessel:test:bulk1/graph?rels=SUPPLIES"); + assert.equal(filtered.body.paths.length, 0); + + assert.equal((await get("/api/intel/entities/vessel:test:bulk1/graph?direction=sideways")).status, 400); + assert.equal((await get("/api/intel/entities/vessel:test:nope/graph")).status, 404); +}); + +test("GET /api/intel/graph/path finds downstream paths", async () => { + const { status, body } = await get("/api/intel/graph/path?from=gen:test:alpha&to=smelter:test:alpha"); + assert.equal(status, 200); + assert.equal(body.paths.length, 1); + assert.equal(body.paths[0].steps[0].rel.rel_id, "rel:alpha-supplies-smelter"); + + assert.equal((await get("/api/intel/graph/path?from=gen:test:alpha")).status, 400); + assert.equal((await get("/api/intel/graph/path?from=gen:test:nope&to=smelter:test:alpha")).status, 404); +}); + +test("GET /api/intel/sources", async () => { + assert.equal((await get("/api/intel/sources")).body.sources.length, 1); + assert.equal((await get("/api/intel/sources/source:test:feed")).body.source.publisher, "Test Publisher"); + assert.equal((await get("/api/intel/sources/source:test:nope")).status, 404); +}); + +test("GET /api/intel/gaps with filters", async () => { + assert.equal((await get("/api/intel/gaps")).body.gaps.length, 1); + assert.equal((await get("/api/intel/gaps?entity=smelter:test:alpha")).body.gaps.length, 1); + assert.equal((await get("/api/intel/gaps?entity=gen:test:alpha")).body.gaps.length, 0); + assert.equal((await get("/api/intel/gaps?priority=P1&status=open")).body.gaps.length, 1); + assert.equal((await get("/api/intel/gaps?priority=P4")).body.gaps.length, 0); +}); + +test("GET /api/intel/inferences and by id", async () => { + const { body } = await get("/api/intel/inferences"); + assert.equal(body.data_class, "derived"); + // smelter-utilisation (smelter) + cargo-inference (vessel) + gap-synthesis (terminal) + assert.equal(body.inferences.length, 3); + const methods = body.inferences.map((i) => i.method).sort(); + assert.deepEqual(methods, [ + "rule:cargo-inference@1.0.0", + "rule:gap-synthesis@1.0.0", + "rule:smelter-utilisation@1.0.0", + ]); + + const vessel = await get("/api/intel/inferences?entity=vessel:test:bulk1"); + assert.equal(vessel.body.inferences.length, 1); + const cargo = vessel.body.inferences[0]; + assert.equal(cargo.claim_structured.predicate, "carries"); + assert.equal(cargo.claim_structured.object, "coal"); + assert.equal(cargo.confidence, 0.85); + + assert.equal((await get("/api/intel/inferences?status=INFERRED")).body.inferences.length, 3); + assert.equal((await get("/api/intel/inferences?status=VERIFIED")).body.inferences.length, 0); + + const byId = await get(`/api/intel/inferences/${cargo.inference_id}`); + assert.equal(byId.status, 200); + assert.equal(byId.body.inference.inference_id, cargo.inference_id); + assert.equal((await get("/api/intel/inferences/infer:2020-01-01:nope-1")).status, 404); +}); + +test("GET /api/intel/sdui/{entityId} composes a schema-valid situation-aware doc", async () => { + const { status, body } = await get("/api/intel/sdui/smelter:test:alpha"); + assert.equal(status, 200); + assert.equal(body.entity_id, "smelter:test:alpha"); + assert.equal(body.entity_type, "Smelter"); + assert.deepEqual(body.situation, ["no-live-data", "has-gaps", "estimated-load"]); + assert.equal(body.layout[0].panel, "gap-card"); // no-live-data hoists the gap card + for (const panel of body.layout) { + if (panel.data !== undefined) assert.match(panel.data.endpoint, /^\/api\/intel\//); + } + + // The vessel HAS live state (position), so no hoisting applies; whether the + // fixture telemetry counts as stale depends on wall clock, so only assert + // the clock-independent parts. + const vessel = await get("/api/intel/sdui/vessel:test:bulk1"); + assert.ok(!vessel.body.situation.includes("no-live-data")); + assert.ok(!vessel.body.situation.includes("has-gaps")); + assert.ok(!vessel.body.situation.includes("anomaly-active")); + assert.equal(vessel.body.layout[0].panel, "headline-state"); + + assert.equal((await get("/api/intel/sdui/vessel:test:nope")).status, 404); +}); + +test("GET /api/intel/meta", async () => { + const { body } = await get("/api/intel/meta"); + assert.deepEqual( + { ...body.counts }, + { entities: 8, relationships: 2, sources: 1, gaps: 1, grids: 1, inferences: 3 }, + ); + assert.equal(body.replay, false); + assert.ok(typeof body.generated_at === "string"); +}); + +test("unknown routes and non-GET methods return 404 JSON", async () => { + const notFound = await get("/api/intel/nope"); + assert.equal(notFound.status, 404); + assert.equal(notFound.body.error, "not_found"); + + const post = await fetch(`${base}/api/intel/grids`, { method: "POST" }); + assert.equal(post.status, 404); +}); + +// ── SSE + replay ─────────────────────────────────────────────────────────── + +test("SSE stream: hello then real replayed state deltas", async () => { + const replayApp = buildApp({ dataDir, varDir, replay: true, replayTickMs: 100 }); + assert.equal(replayApp.replay, true); + assert.equal(replayApp.replayRemaining(), 4); // 7 observations: 3 at boot, 4 queued + const replayServer = await listen(replayApp); + const replayBase = `http://127.0.0.1:${replayServer.address().port}`; + + const controller = new AbortController(); + try { + const res = await fetch(`${replayBase}/api/intel/stream`, { signal: controller.signal }); + assert.equal(res.status, 200); + assert.match(res.headers.get("content-type") ?? "", /^text\/event-stream/); + + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let text = ""; + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + const { value, done } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + if (text.includes("event: state.updated")) break; + } + assert.ok(text.includes("event: hello"), "hello event must arrive first"); + assert.ok(text.includes("event: state.updated"), "a replayed state delta must arrive within 10s"); + const dataLine = text + .split("\n") + .find((l, i, lines) => l.startsWith("data: ") && lines[i - 1] === "event: state.updated"); + const payload = JSON.parse(dataLine.slice("data: ".length)); + assert.equal(payload.data_class, "telemetry"); + assert.ok(payload.state.entity_id.startsWith("region:test:") || payload.state.entity_id.startsWith("intercon:") || payload.state.entity_id.startsWith("vessel:")); + } finally { + controller.abort(); + replayServer.closeAllConnections?.(); + await new Promise((resolve) => replayServer.close(resolve)); + } +}); + +test("SSE stream filters by entities allowlist", async () => { + const controller = new AbortController(); + try { + const res = await fetch(`${base}/api/intel/stream?entities=region:test:north&metrics=grid.demand.mw`, { + signal: controller.signal, + }); + assert.equal(res.status, 200); + const reader = res.body.getReader(); + const { value } = await reader.read(); + const text = new TextDecoder().decode(value); + assert.ok(text.includes("event: hello")); + } finally { + controller.abort(); + } +}); diff --git a/intelligence/packages/api/test/sdui.test.mjs b/intelligence/packages/api/test/sdui.test.mjs new file mode 100644 index 0000000..1c28ba2 --- /dev/null +++ b/intelligence/packages/api/test/sdui.test.mjs @@ -0,0 +1,195 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createValidators } from "@pact-tailor/ontology"; +import { + composePanelDoc, + computeSituations, + applySituationRules, + composeWithAI, + templateFor, +} from "../dist/index.js"; + +const validators = createValidators(); + +function entity(id, type, over = {}) { + return { + entity_id: id, + entity_type: type, + name: `Name of ${id}`, + data_class: "structural", + observability: "KNOWN_LIVE", + sources: ["source:test:feed"], + ...over, + }; +} + +function state(entityId, metric, eventTime) { + return { + entity_id: entityId, + metric, + value: 100, + event_time: eventTime, + ingest_time: eventTime, + source_id: "source:test:feed", + feed_id: "feed:test:main", + quality: "good", + observation_id: "01ARZ3NDEKTSV4RRFFQ69G5FAV", + }; +} + +function gap(entityId) { + return { + gap_id: "gap:test:x", + entity_id: entityId, + desired_metric: "power.load.mw", + commercial_value: "high", + strategic_value: "high", + priority: "P1", + status: "open", + }; +} + +function anomalyInference(subject, status = "INFERRED") { + return { + inference_id: "infer:2026-08-12:anomaly-detection-1", + claim: "anomalous grid.demand.mw", + claim_structured: { subject, predicate: "exhibits_anomaly", object: "grid.demand.mw" }, + tier: "B", + confidence: 0.9, + status, + method: "rule:anomaly-detection@1.0.0", + evidence: [{ kind: "observation", ref: "01ARZ3NDEKTSV4RRFFQ69G5FAV", role: "supports" }], + sources: ["source:test:feed"], + produced_at: "2026-08-12T00:00:00Z", + }; +} + +const NOW = "2026-08-12T12:00:00Z"; + +test("situations: no-live-data, has-gaps, estimated-load", () => { + const e = entity("smelter:test:alpha", "Smelter", { observability: "ESTIMATED" }); + const situations = computeSituations({ entity: e, states: [], gaps: [gap(e.entity_id)], inferences: [], now: NOW }); + assert.deepEqual(situations, ["no-live-data", "has-gaps", "estimated-load"]); +}); + +test("situations: normal when nothing is wrong", () => { + const e = entity("gen:test:alpha", "Generator"); + const fresh = state(e.entity_id, "power.output.mw", "2026-08-12T11:55:00Z"); + const situations = computeSituations({ entity: e, states: [fresh], gaps: [], inferences: [], now: NOW }); + assert.deepEqual(situations, ["normal"]); +}); + +test("situations: stale-telemetry past 3x the 30-minute default cadence", () => { + const e = entity("gen:test:alpha", "Generator"); + // 91 minutes old — beyond 3 x 30 min. + const stale = state(e.entity_id, "power.output.mw", "2026-08-12T10:29:00Z"); + assert.deepEqual( + computeSituations({ entity: e, states: [stale], gaps: [], inferences: [], now: NOW }), + ["stale-telemetry"], + ); + // 89 minutes old — within threshold. + const fresh = state(e.entity_id, "power.output.mw", "2026-08-12T10:31:00Z"); + assert.deepEqual( + computeSituations({ entity: e, states: [fresh], gaps: [], inferences: [], now: NOW }), + ["normal"], + ); +}); + +test("situations: anomaly-active only for open anomaly inferences", () => { + const e = entity("region:test:north", "GridRegion"); + const s = state(e.entity_id, "grid.demand.mw", "2026-08-12T11:59:00Z"); + assert.deepEqual( + computeSituations({ entity: e, states: [s], gaps: [], inferences: [anomalyInference(e.entity_id)], now: NOW }), + ["anomaly-active"], + ); + assert.deepEqual( + computeSituations({ + entity: e, + states: [s], + gaps: [], + inferences: [anomalyInference(e.entity_id, "RETRACTED")], + now: NOW, + }), + ["normal"], + ); + assert.deepEqual( + computeSituations({ + entity: e, + states: [s], + gaps: [], + inferences: [anomalyInference(e.entity_id, "CORROBORATED")], + now: NOW, + }), + ["anomaly-active"], + ); +}); + +test("anomaly-active hoists inference-list to the top", () => { + const e = entity("region:test:north", "GridRegion"); + const doc = composePanelDoc({ + entity: e, + states: [state(e.entity_id, "grid.demand.mw", "2026-08-12T11:59:00Z")], + gaps: [], + inferences: [anomalyInference(e.entity_id)], + now: NOW, + }); + assert.deepEqual(doc.situation, ["anomaly-active"]); + assert.equal(doc.layout[0].panel, "inference-list"); +}); + +test("no-live-data hoists (or injects) the gap-card; anomaly still wins the top slot", () => { + const e = entity("smelter:test:alpha", "Smelter"); + const doc = composePanelDoc({ entity: e, states: [], gaps: [gap(e.entity_id)], inferences: [], now: NOW }); + assert.equal(doc.layout[0].panel, "gap-card"); + + const both = applySituationRules( + templateFor("Smelter")(e.entity_id), + ["no-live-data", "anomaly-active"], + e.entity_id, + ); + assert.equal(both[0].panel, "inference-list"); + assert.equal(both[1].panel, "gap-card"); +}); + +test("has-gaps injects a gap-card into templates that lack one", () => { + const e = entity("gen:test:alpha", "Generator"); + const base = templateFor("Generator")(e.entity_id); + assert.ok(!base.some((p) => p.panel === "gap-card")); + const doc = composePanelDoc({ + entity: e, + states: [state(e.entity_id, "power.output.mw", "2026-08-12T11:59:00Z")], + gaps: [gap(e.entity_id)], + inferences: [], + now: NOW, + }); + assert.ok(doc.layout.some((p) => p.panel === "gap-card")); +}); + +test("composed docs validate against the SDUI panel schema and never inline data", () => { + const cases = [ + entity("grid:test", "Grid"), + entity("region:test:north", "GridRegion"), + entity("gen:test:alpha", "Generator"), + entity("smelter:test:alpha", "Smelter", { observability: "ESTIMATED" }), + entity("vessel:test:bulk1", "Vessel"), + entity("port:test:main", "Port"), + entity("term:test:coal", "Terminal"), + entity("mine:test:black", "Mine"), + entity("company:test:acme", "Company"), // default template + ]; + for (const e of cases) { + const doc = composePanelDoc({ entity: e, states: [], gaps: [gap(e.entity_id)], inferences: [], now: NOW }); + const result = validators.sduiPanel(doc); + assert.ok(result.ok, `sdui doc for ${e.entity_type} must validate: ${result.ok ? "" : result.errors.join("; ")}`); + for (const panel of doc.layout) { + if (panel.data !== undefined) assert.match(panel.data.endpoint, /^\/api\/intel\//); + assert.ok(!("values" in (panel.props ?? {})), "panels must not inline data values"); + } + } +}); + +test("composeWithAI is the documented identity stub", () => { + const e = entity("gen:test:alpha", "Generator"); + const layout = templateFor("Generator")(e.entity_id); + assert.equal(composeWithAI(e, ["normal"], layout), layout); +}); From a7f13c6617784194ad8635ba0555d5bbce145dce Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 02:05:38 +0000 Subject: [PATCH 15/16] feat(intelligence): deck.gl GPU globe prototype with LOD tiers (phases 19-21) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _GlobeView + instanced layers only — no DOM for geographic objects: land basemap (bundled hand-simplified outline, production swaps Natural Earth 110m), grid markers, animated interconnector arcs, capacity/utilisation- encoded generators, heading-rotated vessels, and pulsing observability-gap rings as a first-class visual. Zoom-gated LOD (global → continental → regional → facility) with the SDUI side panel rendering all five panel kinds and a 3DGS mount placeholder. SSE state cache with reconnect backoff; degrades to structural view when the API is absent; every data view carries a replayed-not-live banner. Zero external requests at build and runtime. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014yVGqBtvM1zNAd9tvUVNzm --- intelligence/packages/globe/assets/README.md | 14 + .../globe/assets/ne_110m_land.geojson | 1 + intelligence/packages/globe/index.html | 13 + intelligence/packages/globe/src/api.ts | 108 ++++ intelligence/packages/globe/src/basemap.ts | 47 ++ intelligence/packages/globe/src/hud.ts | 48 ++ intelligence/packages/globe/src/layers.ts | 468 ++++++++++++++++ intelligence/packages/globe/src/lod.ts | 94 ++++ intelligence/packages/globe/src/main.ts | 189 ++++++- intelligence/packages/globe/src/panel.ts | 515 +++++++++++++++++ .../packages/globe/src/state-cache.ts | 227 ++++++++ intelligence/packages/globe/src/style.css | 524 ++++++++++++++++++ intelligence/packages/globe/src/types.ts | 234 ++++++++ intelligence/packages/globe/src/vite-env.d.ts | 1 + intelligence/packages/globe/vite.config.ts | 22 + 15 files changed, 2503 insertions(+), 2 deletions(-) create mode 100644 intelligence/packages/globe/assets/README.md create mode 100644 intelligence/packages/globe/assets/ne_110m_land.geojson create mode 100644 intelligence/packages/globe/index.html create mode 100644 intelligence/packages/globe/src/api.ts create mode 100644 intelligence/packages/globe/src/basemap.ts create mode 100644 intelligence/packages/globe/src/hud.ts create mode 100644 intelligence/packages/globe/src/layers.ts create mode 100644 intelligence/packages/globe/src/lod.ts create mode 100644 intelligence/packages/globe/src/panel.ts create mode 100644 intelligence/packages/globe/src/state-cache.ts create mode 100644 intelligence/packages/globe/src/style.css create mode 100644 intelligence/packages/globe/src/types.ts create mode 100644 intelligence/packages/globe/src/vite-env.d.ts create mode 100644 intelligence/packages/globe/vite.config.ts diff --git a/intelligence/packages/globe/assets/README.md b/intelligence/packages/globe/assets/README.md new file mode 100644 index 0000000..4db945c --- /dev/null +++ b/intelligence/packages/globe/assets/README.md @@ -0,0 +1,14 @@ +# Basemap assets + +`ne_110m_land.geojson` is a **hand-simplified land outline** authored for this +workspace (egress is blocked here, so the real dataset cannot be fetched at +build time). It is coarse but recognisable, with extra vertex budget spent on +the Australian coastline — the Queensland coast and the Gladstone harbour +notch — because the demo scene focuses there. + +Production swaps this file for the real **Natural Earth 1:110m land** +(`ne_110m_land`) GeoJSON, which is public domain: + + +The file is imported at build time (`?raw`) and bundled — the globe renders +with zero external tile, font, or data requests. diff --git a/intelligence/packages/globe/assets/ne_110m_land.geojson b/intelligence/packages/globe/assets/ne_110m_land.geojson new file mode 100644 index 0000000..e7c6756 --- /dev/null +++ b/intelligence/packages/globe/assets/ne_110m_land.geojson @@ -0,0 +1 @@ +{"type":"FeatureCollection","name":"simplified_land_outline","features":[{"type":"Feature","properties":{"name":"Australia"},"geometry":{"type":"Polygon","coordinates":[[[142.5,-10.7],[143.6,-14.5],[145.3,-15],[145.9,-16.9],[146.3,-18.3],[146.9,-19.3],[148.2,-19.9],[149.2,-21.1],[149.7,-22.4],[150.8,-22.7],[151.2,-23.5],[151.1,-23.85],[151.35,-24],[151.9,-24.6],[152.5,-25.3],[153.15,-25.9],[153.1,-26.8],[153.45,-27.5],[153.6,-28.7],[153,-30.5],[152,-32.6],[151.2,-33.9],[150.6,-35.3],[150,-37.4],[147.9,-37.9],[146.4,-39.1],[145.4,-38.3],[144.9,-38.4],[144.9,-37.9],[144.5,-38.3],[143.5,-38.8],[141,-38.3],[139.7,-37.2],[139.5,-36],[138.5,-35.6],[138.5,-34.6],[138.1,-34.3],[137.9,-35.1],[137.8,-33.4],[137,-33.7],[136,-34.9],[135.5,-34.6],[134.2,-32.7],[131.1,-31.5],[127,-32.3],[123.5,-33.9],[121,-33.9],[118,-35],[115,-34.4],[115.7,-33.2],[115.7,-32],[114.9,-30],[114.1,-28],[113.4,-26.1],[113.8,-23.4],[114.1,-21.8],[115.5,-21],[116.7,-20.6],[119,-20],[121,-19.5],[122.2,-18],[123,-16.4],[124.5,-16.4],[125.2,-14.5],[126.9,-13.8],[128.2,-15.2],[129.5,-14.9],[129.8,-13.5],[130.3,-12.5],[131.9,-12.2],[132.6,-11.4],[135.2,-12.1],[136.8,-12.2],[135.9,-13.2],[135.7,-14.9],[136.8,-15.9],[139,-17.6],[140.8,-17.5],[141.4,-16.2],[141.6,-12.9],[142.1,-11.1],[142.5,-10.7]]]}},{"type":"Feature","properties":{"name":"Tasmania"},"geometry":{"type":"Polygon","coordinates":[[[144.7,-40.7],[147,-40.8],[148.3,-40.9],[148.3,-42.2],[147.9,-43.2],[147,-43.6],[145.9,-43.6],[145.2,-42.2],[144.7,-41.2],[144.7,-40.7]]]}},{"type":"Feature","properties":{"name":"New Guinea"},"geometry":{"type":"Polygon","coordinates":[[[131.2,-1.4],[134.2,-2.5],[136.5,-1.8],[140,-2.4],[143.5,-3.6],[145.8,-4.9],[147.5,-5.9],[148.8,-8],[150.6,-10.4],[147.2,-8.1],[144.5,-7.7],[143,-8.5],[141,-8.1],[138.9,-8.2],[138.5,-6.5],[135.3,-4.5],[133.5,-3.9],[132.4,-4.1],[131.9,-2.7],[131.2,-1.4]]]}},{"type":"Feature","properties":{"name":"New Zealand"},"geometry":{"type":"MultiPolygon","coordinates":[[[[172.8,-34.4],[174.3,-35.3],[175.4,-36.5],[176,-37.6],[178,-37.5],[178.5,-38.6],[177,-39.5],[176.9,-40.2],[175.3,-41.6],[174.6,-39.3],[174.8,-38],[173,-35.5],[172.8,-34.4]]],[[[172.7,-40.5],[174.3,-41],[174.2,-41.7],[173.1,-42.9],[172.7,-43.6],[171.2,-44.3],[170.7,-45.9],[168.4,-46.6],[166.5,-46],[168.4,-44],[170.6,-42.9],[172.1,-41],[172.7,-40.5]]]]}},{"type":"Feature","properties":{"name":"Maritime SE Asia"},"geometry":{"type":"MultiPolygon","coordinates":[[[[95.3,5.6],[98.3,3.7],[100.4,1.7],[103,0.5],[104.4,-1.5],[106,-3],[105.8,-5.9],[104.5,-5.5],[102,-4],[100.3,-1],[98.6,1.7],[96,3.7],[95.3,5.6]]],[[[105.2,-6.8],[107,-6.2],[110.4,-6.5],[112.7,-6.9],[114.4,-7.7],[114.6,-8.7],[111.5,-8.3],[108,-7.8],[105.6,-7.5],[105.2,-6.8]]],[[[109.6,1.9],[111.4,2.9],[113,4.6],[115.2,5.6],[117.6,6.9],[119.2,5.3],[117.6,3.6],[117.5,0.3],[116.5,-2.5],[114.5,-3.5],[110.2,-3],[109,-0.9],[108.9,0.9],[109.6,1.9]]],[[[119.5,0.8],[120.5,1.3],[123.9,0.8],[121.5,-0.9],[123.2,-2],[122.3,-4.5],[120.4,-5.6],[119.4,-5.6],[119.5,-3.5],[118.8,-2.7],[119.8,-0.7],[119.5,0.8]]],[[[120.3,18.6],[122.2,17],[124,13],[123,9],[126.6,7.2],[125.5,5.6],[122,6.9],[121.9,10.5],[120,14],[119.8,16.3],[120.3,18.6]]]]}},{"type":"Feature","properties":{"name":"Africa"},"geometry":{"type":"Polygon","coordinates":[[[-5.9,35.8],[3,36.9],[10.2,37.2],[15.3,32.4],[20.1,32.2],[25,31.6],[31.1,31.5],[34.2,27.8],[37.5,18.7],[43.4,12.5],[45,10.9],[51.4,11.8],[47.5,4.6],[41,-1.9],[40.5,-10.5],[40.5,-15.5],[35,-20],[35.5,-24],[32.6,-28.8],[27.9,-33],[20,-34.8],[18.3,-33.9],[17.1,-29.5],[14.5,-22.5],[11.8,-18],[13.4,-12.5],[12.3,-6.1],[9.7,-1.6],[9.3,3.9],[5.9,4.3],[1.2,6.1],[-4,5.2],[-7.6,4.4],[-13.3,8.5],[-17.4,14.7],[-16.5,19.5],[-14.8,23.5],[-13,27.7],[-9.6,30.4],[-9.8,33.9],[-5.9,35.8]]]}},{"type":"Feature","properties":{"name":"Eurasia"},"geometry":{"type":"Polygon","coordinates":[[[-9.3,38.7],[-8.9,43.3],[-4.8,43.4],[-1.2,46.2],[-4.6,48.4],[1.4,50.9],[4,52],[8,53.6],[8.6,56.2],[10.6,59.5],[5.5,60.5],[12,65.2],[18,69.5],[25.8,71.1],[31,69.9],[40.4,67.8],[44.2,66.7],[54,68.5],[66,69],[73.5,71.5],[80,73],[95,76.1],[105,77.5],[113,73.8],[125,73],[140,72.2],[151,70.9],[160,69.7],[170,69.8],[178.5,65.5],[174,61.7],[165,60.1],[162.5,56.5],[156.7,50.9],[155.5,56.5],[152.5,59],[143,53.5],[138.5,48],[131.9,43.1],[129.6,40.1],[129.5,35.5],[126.5,34.4],[126.6,37.7],[124.2,39.8],[121.5,38.9],[117.7,38.7],[120.3,34.3],[121.8,31.2],[120,26.5],[116.5,23.3],[113.5,22.2],[110,21.4],[105.9,20],[109,15.5],[108,10.9],[104.8,8.6],[100.5,13.6],[102,11.5],[103.5,1.4],[100.5,5.5],[98.3,8.4],[97.6,16],[94.5,17.8],[91.5,22.5],[88,21.7],[86,20.2],[80.3,15.8],[80.3,13.1],[79.9,10.3],[77.5,8.1],[74.9,12.9],[72.8,19],[72.6,22.3],[69,22.2],[66.9,24.8],[61.6,25.2],[57,25.6],[51.6,24.2],[48.6,28.6],[50,26.5],[51.5,25.3],[52.6,24],[56.4,26.2],[58.8,23.6],[59.8,22.2],[57.8,19],[53.1,16.7],[45.1,12.7],[43.3,12.7],[42.7,15],[38.9,21.5],[35,28],[32.6,29.9],[34.2,31.3],[35.9,35.9],[36.2,36.6],[30.5,36.3],[27.3,36.7],[26.3,38.4],[26.7,40.4],[24.9,40.9],[22.6,40],[23,38],[23.2,36.4],[21.3,37.6],[19.4,40.3],[18.5,42.5],[13.6,45.6],[12.5,44.2],[15.7,41.9],[18.4,40.3],[16.6,38.9],[15.6,38],[16,40],[13.7,41.2],[12.2,41.7],[10.5,42.9],[8.9,44.4],[6.5,43.1],[3,43.2],[3.2,41.9],[0.2,39.6],[-0.7,37.6],[-2.1,36.7],[-5.4,36.1],[-7,37.2],[-9.3,38.7]]]}},{"type":"Feature","properties":{"name":"Britain & Ireland"},"geometry":{"type":"MultiPolygon","coordinates":[[[[-5.7,50.1],[-1.3,50.8],[1.4,51.4],[1.7,52.7],[-0.2,54.1],[-1.6,55.6],[-2.1,57.7],[-5,58.6],[-5.8,56.5],[-4.8,54.9],[-3.1,54.2],[-4.7,53.4],[-5.3,51.7],[-3,51.2],[-4.5,50.4],[-5.7,50.1]]],[[[-6.2,54.5],[-6,52.3],[-8.5,51.5],[-10.3,51.8],[-9.9,53.5],[-8.5,55.2],[-6.2,54.5]]]]}},{"type":"Feature","properties":{"name":"Japan"},"geometry":{"type":"Polygon","coordinates":[[[141.9,45.5],[145.3,44.3],[143.3,42],[140.9,41.6],[141.5,38.4],[140.9,35.7],[138.9,34.7],[137,34.7],[135.1,33.6],[132.9,32.8],[131.4,31.4],[130.2,32.8],[129.6,33.4],[132.4,35.5],[136.7,37.3],[139.9,40.1],[140.3,43.2],[141.9,45.5]]]}},{"type":"Feature","properties":{"name":"North America"},"geometry":{"type":"Polygon","coordinates":[[[-166.5,68.9],[-161,70.3],[-156.5,71.3],[-145,70.1],[-135,69.4],[-125,69.9],[-115,68.5],[-107,68],[-96,68.2],[-90,68.5],[-85.5,66.5],[-87,63.5],[-94.2,58.7],[-92.5,55.9],[-85,55.3],[-82.2,52.9],[-79.5,51.3],[-78.5,55.5],[-77.5,60.5],[-71.5,61.5],[-64.5,60.3],[-58.5,54.5],[-55.7,51.5],[-60,46.9],[-66,44.7],[-70.2,42],[-74,40.5],[-75.5,35.3],[-81,31.9],[-80.1,25.8],[-81.8,24.6],[-82.7,27.9],[-84.4,30],[-89.1,30.2],[-94,29.7],[-97.2,26],[-97.8,22.3],[-95.9,18.9],[-91.5,18.7],[-90.4,21.3],[-87,21.5],[-88.3,18.5],[-88.2,15.7],[-83.2,15],[-83.6,10.9],[-79.5,9.6],[-77.2,8.6],[-78.2,8.4],[-80,7.5],[-85.7,10],[-87.9,13.1],[-91.5,14],[-95.2,16.2],[-105,19.5],[-106.5,23.2],[-109.5,25.8],[-112,29],[-114.6,31.7],[-112.8,28.3],[-109.8,22.9],[-115,27.8],[-117.1,32.5],[-120.6,34.6],[-122.4,37.8],[-124.2,40.4],[-124.7,48.4],[-128,51.5],[-133,55],[-139.5,59.5],[-146.5,61],[-151.5,59.2],[-156.5,57],[-163.5,54.8],[-160,58.5],[-165.5,61.5],[-161,64.5],[-168.1,65.6],[-166.5,68.9]]]}},{"type":"Feature","properties":{"name":"South America"},"geometry":{"type":"Polygon","coordinates":[[[-77.2,8.6],[-75.3,10.8],[-71.5,12.4],[-68,10.6],[-63,10.7],[-60,8.5],[-55,6],[-51.5,4.3],[-49.9,0.1],[-44.3,-2.6],[-39,-4.6],[-34.8,-7],[-37,-11],[-38.9,-13],[-39.8,-17.8],[-40.9,-21.9],[-43.2,-23],[-46.4,-24],[-48.6,-26],[-51.5,-31],[-53.5,-34],[-56,-34.9],[-58.4,-34.6],[-57.1,-36.3],[-62,-38.9],[-65.1,-40.8],[-63.6,-42.5],[-65.3,-45],[-67.5,-46],[-65.9,-47.1],[-68.4,-50.1],[-69.2,-52.2],[-68.4,-54.9],[-72,-53.7],[-73.5,-50],[-73.7,-46.5],[-73,-42],[-73.2,-39.5],[-71.6,-33],[-70.4,-23.6],[-70.3,-18.4],[-75.9,-14.7],[-77.2,-12.1],[-79.7,-7.1],[-81.3,-5.9],[-80.9,-1],[-77.9,1.5],[-77.4,4],[-78.6,7.2],[-77.2,8.6]]]}},{"type":"Feature","properties":{"name":"Greenland"},"geometry":{"type":"Polygon","coordinates":[[[-45,59.9],[-49.5,62.5],[-52,66],[-54,70.5],[-56.5,74.5],[-61,76.2],[-68,78.5],[-58.5,81.5],[-44,83.2],[-30,82],[-21,79.5],[-18,76.5],[-22,70.5],[-25.5,68.5],[-32.5,67.5],[-38,65.5],[-42,62],[-45,59.9]]]}},{"type":"Feature","properties":{"name":"Madagascar"},"geometry":{"type":"Polygon","coordinates":[[[49.3,-12.1],[50.2,-15.5],[49.8,-17],[47.2,-24.8],[45.2,-25.6],[43.3,-22.3],[44,-19.9],[44.5,-16.2],[46.3,-13.8],[49.3,-12.1]]]}},{"type":"Feature","properties":{"name":"Iceland"},"geometry":{"type":"Polygon","coordinates":[[[-24.5,65.5],[-21.5,66.4],[-16.5,66.5],[-13.6,65.5],[-15,64.3],[-18.7,63.4],[-22.7,63.8],[-24.5,65.5]]]}},{"type":"Feature","properties":{"name":"Sri Lanka"},"geometry":{"type":"Polygon","coordinates":[[[79.9,9.8],[81.9,8.6],[81.7,6.5],[80.6,5.9],[79.7,8.1],[79.9,9.8]]]}}]} diff --git a/intelligence/packages/globe/index.html b/intelligence/packages/globe/index.html new file mode 100644 index 0000000..cd7efe2 --- /dev/null +++ b/intelligence/packages/globe/index.html @@ -0,0 +1,13 @@ + + + + + + + PACT.TAILOR — globe + + +
+ + + diff --git a/intelligence/packages/globe/src/api.ts b/intelligence/packages/globe/src/api.ts new file mode 100644 index 0000000..ff40bb4 --- /dev/null +++ b/intelligence/packages/globe/src/api.ts @@ -0,0 +1,108 @@ +// Typed client for the pinned /api/intel contract. Every call is wrapped: +// failures resolve to null and flip the app-global `apiDown` flag — the globe +// must degrade to a structural view when the API is absent, never crash. + +import type { + EntitiesResponse, + EntityStateResponse, + GapsResponse, + GridsResponse, + GridSummaryResponse, + SduiPanelDoc, +} from "./types.js"; + +export const API_BASE = "/api/intel"; + +type StatusListener = (down: boolean) => void; + +const statusListeners = new Set(); + +/** App-global API health flag. `down` = last request failed at network level. */ +export const apiStatus = { down: false }; + +function setDown(down: boolean): void { + if (apiStatus.down === down) return; + apiStatus.down = down; + // Mirror for quick inspection from the console. + (globalThis as Record).apiDown = down; + for (const fn of statusListeners) fn(down); +} + +export function onApiStatusChange(fn: StatusListener): () => void { + statusListeners.add(fn); + return () => statusListeners.delete(fn); +} + +async function request(path: string): Promise { + try { + const res = await fetch(path, { headers: { accept: "application/json" } }); + if (!res.ok) { + // The server answered — API is up, this resource just isn't there. + setDown(false); + return null; + } + const body = (await res.json()) as T; + setDown(false); + return body; + } catch { + setDown(true); + return null; + } +} + +export interface EntityQuery { + type?: string; + grid?: string; + /** "minLon,minLat,maxLon,maxLat" */ + bbox?: string; + q?: string; +} + +export const api = { + getGrids(): Promise { + return request(`${API_BASE}/grids`); + }, + + getGridSummary(gridId: string): Promise { + return request( + `${API_BASE}/grids/${encodeURIComponent(gridId)}/summary`, + ); + }, + + getEntities(query: EntityQuery = {}): Promise { + const params = new URLSearchParams(); + if (query.type) params.set("type", query.type); + if (query.grid) params.set("grid", query.grid); + if (query.bbox) params.set("bbox", query.bbox); + if (query.q) params.set("q", query.q); + const qs = params.toString(); + return request(`${API_BASE}/entities${qs ? `?${qs}` : ""}`); + }, + + getEntityState(entityId: string): Promise { + return request( + `${API_BASE}/entities/${encodeURIComponent(entityId)}/state`, + ); + }, + + getGaps(entityId?: string): Promise { + const qs = entityId ? `?entity=${encodeURIComponent(entityId)}` : ""; + return request(`${API_BASE}/gaps${qs}`); + }, + + getSdui(entityId: string): Promise { + return request( + `${API_BASE}/sdui/${encodeURIComponent(entityId)}`, + ); + }, + + /** + * Fetch an arbitrary endpoint referenced by an SDUI panel `data` block. + * SDUI rule: panels reference endpoints, never inline values — the client + * always fetches. Only same-origin /api/intel endpoints are honoured. + */ + fetchEndpoint(endpoint: string): Promise { + if (!endpoint.startsWith(API_BASE)) return Promise.resolve(null); + return request(endpoint); + }, +}; diff --git a/intelligence/packages/globe/src/basemap.ts b/intelligence/packages/globe/src/basemap.ts new file mode 100644 index 0000000..1be428b --- /dev/null +++ b/intelligence/packages/globe/src/basemap.ts @@ -0,0 +1,47 @@ +// Bundled basemap: simplified land-outline GeoJSON committed under assets/, +// rendered as a dim GeoJsonLayer plus a solid "ocean" sphere backdrop so the +// globe reads against the space background with zero external requests. + +import { GeoJsonLayer, SolidPolygonLayer } from "@deck.gl/layers"; +import type { FeatureCollection } from "geojson"; +import landRaw from "../assets/ne_110m_land.geojson?raw"; + +export const LAND: FeatureCollection = JSON.parse(landRaw) as FeatureCollection; + +/** Whole-world polygon — renders as the ocean sphere under _GlobeView. */ +const WORLD: [number, number][][] = [ + [ + [-180, 90], + [0, 90], + [180, 90], + [180, -90], + [0, -90], + [-180, -90], + ], +]; + +export function buildOceanLayer(): SolidPolygonLayer<[number, number][]> { + return new SolidPolygonLayer<[number, number][]>({ + id: "ocean-sphere", + data: WORLD, + getPolygon: (d) => d, + filled: true, + getFillColor: [8, 14, 30, 255], + pickable: false, + }); +} + +export function buildLandLayer(): GeoJsonLayer { + return new GeoJsonLayer({ + id: "basemap-land", + data: LAND, + filled: true, + stroked: true, + getFillColor: [30, 41, 59, 235], + getLineColor: [71, 85, 105, 180], + getLineWidth: 1, + lineWidthUnits: "pixels", + lineWidthMinPixels: 1, + pickable: false, + }); +} diff --git a/intelligence/packages/globe/src/hud.ts b/intelligence/packages/globe/src/hud.ts new file mode 100644 index 0000000..abdef18 --- /dev/null +++ b/intelligence/packages/globe/src/hud.ts @@ -0,0 +1,48 @@ +// Top-left HUD: title, LOD tier, connection status, entity count. +// DOM is reserved for HUD + side panel — never for geographic objects. + +import type { LodTier } from "./lod.js"; + +export type ConnectionKind = "live" | "offline" | "connecting"; + +function el(tag: string, className: string, text?: string): HTMLElement { + const node = document.createElement(tag); + node.className = className; + if (text !== undefined) node.textContent = text; + return node; +} + +export class Hud { + private tierEl: HTMLElement; + private connEl: HTMLElement; + private countEl: HTMLElement; + + constructor(parent: HTMLElement) { + const root = el("div", "hud"); + root.appendChild( + el("div", "hud-title", "PACT.TAILOR — physical-economy intelligence"), + ); + const row = el("div", "hud-row"); + this.tierEl = el("span", "hud-chip hud-tier", "global"); + this.connEl = el("span", "hud-chip hud-conn is-connecting", "connecting…"); + this.countEl = el("span", "hud-chip hud-count", "0 entities"); + row.appendChild(this.tierEl); + row.appendChild(this.connEl); + row.appendChild(this.countEl); + root.appendChild(row); + parent.appendChild(root); + } + + setTier(tier: LodTier): void { + this.tierEl.textContent = tier; + } + + setConnection(kind: ConnectionKind, text: string): void { + this.connEl.textContent = text; + this.connEl.className = `hud-chip hud-conn is-${kind}`; + } + + setEntityCount(n: number): void { + this.countEl.textContent = `${n.toLocaleString()} ${n === 1 ? "entity" : "entities"}`; + } +} diff --git a/intelligence/packages/globe/src/layers.ts b/intelligence/packages/globe/src/layers.ts new file mode 100644 index 0000000..62ca0d2 --- /dev/null +++ b/intelligence/packages/globe/src/layers.ts @@ -0,0 +1,468 @@ +// GPU layer builders. Everything geographic is instanced — no DOM markers. + +import type { Color } from "@deck.gl/core"; +import { ArcLayer, IconLayer, ScatterplotLayer } from "@deck.gl/layers"; +import type { StateCache } from "./state-cache.js"; +import type { EntityRecord, ObservabilityGapRecord } from "./types.js"; + +// --------------------------------------------------------------------------- +// Shared helpers +// --------------------------------------------------------------------------- + +/** NEM region centroids — the demo's hardcoded interconnector anchor points. */ +export const NEM_REGION_CENTROIDS: Record = { + QLD1: [146.5, -21], + NSW1: [147, -32.5], + VIC1: [144.5, -36.8], + SA1: [137, -31], + TAS1: [146.5, -42], +}; + +export function pointCoords(e: EntityRecord): [number, number] | null { + const g = e.geometry; + if (!g || g.type !== "Point") return null; + const c = g.coordinates as number[]; + const lon = c[0]; + const lat = c[1]; + if (typeof lon !== "number" || typeof lat !== "number") return null; + return [lon, lat]; +} + +const GREY: Color = [107, 114, 128, 200]; + +/** Utilisation colour ramp: cool (idle) → hot (flat out). */ +const RAMP: Array<[number, Color]> = [ + [0.0, [56, 130, 246, 220]], + [0.35, [45, 212, 191, 225]], + [0.7, [250, 204, 21, 230]], + [1.0, [248, 113, 113, 240]], +]; + +function rampColor(t: number): Color { + const x = Math.max(0, Math.min(1, t)); + for (let i = 1; i < RAMP.length; i++) { + const hi = RAMP[i]; + const lo = RAMP[i - 1]; + if (!hi || !lo) break; + if (x <= hi[0]) { + const span = hi[0] - lo[0] || 1; + const f = (x - lo[0]) / span; + const a = lo[1]; + const b = hi[1]; + return [ + Math.round((a[0] ?? 0) + ((b[0] ?? 0) - (a[0] ?? 0)) * f), + Math.round((a[1] ?? 0) + ((b[1] ?? 0) - (a[1] ?? 0)) * f), + Math.round((a[2] ?? 0) + ((b[2] ?? 0) - (a[2] ?? 0)) * f), + Math.round((a[3] ?? 255) + ((b[3] ?? 255) - (a[3] ?? 255)) * f), + ]; + } + } + const last = RAMP[RAMP.length - 1]; + return last ? last[1] : GREY; +} + +const GEN_MW_METRICS = [ + "power.output_mw", + "generation_mw", + "power.generation_mw", + "output_mw", + "dispatch_mw", + "mw", +]; + +const FLOW_MW_METRICS = [ + "flow_mw", + "power.flow_mw", + "net_flow_mw", + "interconnector.flow_mw", + "mw", +]; + +function liveMegawatts( + cache: StateCache, + entityId: string, + candidates: string[], +): number | undefined { + return ( + cache.getFirstNumber(entityId, candidates) ?? cache.findMegawatts(entityId) + ); +} + +// --------------------------------------------------------------------------- +// Icon atlas (runtime canvas — no network, no bundled binaries) +// --------------------------------------------------------------------------- + +const ICON = 64; + +export const ICON_MAPPING: Record< + string, + { x: number; y: number; width: number; height: number; mask: boolean } +> = { + triangle: { x: 0, y: 0, width: ICON, height: ICON, mask: true }, + square: { x: ICON, y: 0, width: ICON, height: ICON, mask: true }, + diamond: { x: ICON * 2, y: 0, width: ICON, height: ICON, mask: true }, +}; + +let atlasUrl: string | null = null; + +/** White shapes on transparent — used as alpha masks and tinted per instance. */ +function getIconAtlas(): string { + if (atlasUrl) return atlasUrl; + const canvas = document.createElement("canvas"); + canvas.width = ICON * 3; + canvas.height = ICON; + const ctx = canvas.getContext("2d"); + if (!ctx) return ""; + ctx.fillStyle = "#ffffff"; + // triangle (points up — rotated per-vessel by heading) + ctx.beginPath(); + ctx.moveTo(32, 4); + ctx.lineTo(56, 58); + ctx.lineTo(32, 46); + ctx.lineTo(8, 58); + ctx.closePath(); + ctx.fill(); + // square + ctx.fillRect(ICON + 14, 14, 36, 36); + // diamond + ctx.beginPath(); + ctx.moveTo(ICON * 2 + 32, 6); + ctx.lineTo(ICON * 2 + 58, 32); + ctx.lineTo(ICON * 2 + 32, 58); + ctx.lineTo(ICON * 2 + 6, 32); + ctx.closePath(); + ctx.fill(); + atlasUrl = canvas.toDataURL("image/png"); + return atlasUrl; +} + +// --------------------------------------------------------------------------- +// Grid markers +// --------------------------------------------------------------------------- + +export interface GridMarker { + id: string; + name: string; + position: [number, number]; + entity?: EntityRecord; +} + +/** + * Markers for grids/regions. Entities with geometry win; the hardcoded NEM + * centroids fill in so the globe shows structure even with the API offline. + */ +export function collectGridMarkers(entities: EntityRecord[]): GridMarker[] { + const markers = new Map(); + for (const [region, position] of Object.entries(NEM_REGION_CENTROIDS)) { + markers.set(region, { id: `nem-region:${region}`, name: region, position }); + } + for (const e of entities) { + if (e.entity_type !== "Grid" && e.entity_type !== "GridRegion") continue; + const position = pointCoords(e); + if (!position) continue; + const regionKey = Object.keys(NEM_REGION_CENTROIDS).find( + (r) => e.entity_id.includes(r.toLowerCase()) || e.name.includes(r), + ); + markers.set(regionKey ?? e.entity_id, { + id: e.entity_id, + name: e.name, + position, + entity: e, + }); + } + return [...markers.values()]; +} + +export function buildGridMarkersLayer( + markers: GridMarker[], +): ScatterplotLayer { + return new ScatterplotLayer({ + id: "grid-markers", + data: markers, + getPosition: (d) => d.position, + getRadius: 45_000, + radiusMinPixels: 3, + radiusMaxPixels: 10, + filled: true, + stroked: true, + getFillColor: [148, 163, 184, 90], + getLineColor: [203, 213, 225, 190], + lineWidthMinPixels: 1, + pickable: true, + }); +} + +// --------------------------------------------------------------------------- +// Generators +// --------------------------------------------------------------------------- + +function generatorCapacity(e: EntityRecord): number { + const cap = e.properties?.["capacity_mw"]; + return typeof cap === "number" && cap > 0 ? cap : 0; +} + +export function buildGeneratorsLayer( + generators: EntityRecord[], + cache: StateCache, + opts: { minCapacityMw?: number } = {}, +): ScatterplotLayer { + const min = opts.minCapacityMw ?? 0; + const data = generators.filter( + (e) => pointCoords(e) !== null && generatorCapacity(e) >= min, + ); + return new ScatterplotLayer({ + id: min > 0 ? "generators-major" : "generators", + data, + getPosition: (d) => pointCoords(d) ?? [0, 0], + // radius = sqrt(capacity): area tracks capacity + getRadius: (d) => Math.sqrt(Math.max(generatorCapacity(d), 1)), + radiusScale: 500, + radiusUnits: "meters", + radiusMinPixels: 2.5, + radiusMaxPixels: 26, + filled: true, + stroked: true, + lineWidthMinPixels: 1, + getLineColor: [15, 23, 42, 160], + getFillColor: (d) => { + const cap = generatorCapacity(d); + const mw = liveMegawatts(cache, d.entity_id, GEN_MW_METRICS); + if (cap <= 0 || mw === undefined) return GREY; // no live state → grey + return rampColor(mw / cap); + }, + pickable: true, + updateTriggers: { getFillColor: cache.version }, + }); +} + +// --------------------------------------------------------------------------- +// Interconnectors +// --------------------------------------------------------------------------- + +interface InterconnectorArc { + entity: EntityRecord; + source: [number, number]; + target: [number, number]; +} + +function regionCentroid(region: unknown): [number, number] | null { + if (typeof region !== "string") return null; + return NEM_REGION_CENTROIDS[region.toUpperCase()] ?? null; +} + +const FLOW_FORWARD: Color = [56, 189, 248, 210]; // cyan: from → to +const FLOW_REVERSE: Color = [251, 146, 60, 210]; // amber: to → from +const FLOW_NONE: Color = [100, 116, 139, 130]; + +export function buildInterconnectorsLayer( + interconnectors: EntityRecord[], + cache: StateCache, +): ArcLayer { + const data: InterconnectorArc[] = []; + for (const e of interconnectors) { + const source = regionCentroid(e.properties?.["from_region"]); + const target = regionCentroid(e.properties?.["to_region"]); + if (source && target) data.push({ entity: e, source, target }); + } + const flowOf = (d: InterconnectorArc): number => + liveMegawatts(cache, d.entity.entity_id, FLOW_MW_METRICS) ?? 0; + return new ArcLayer({ + id: "interconnectors", + data, + getSourcePosition: (d) => d.source, + getTargetPosition: (d) => d.target, + getHeight: 0.35, + greatCircle: false, + // width by |flow|; a slim structural arc when no live flow + getWidth: (d) => { + const mw = Math.abs(flowOf(d)); + return mw > 0 ? Math.max(1.5, Math.sqrt(mw) / 3) : 1.2; + }, + widthUnits: "pixels", + widthMinPixels: 1, + widthMaxPixels: 14, + getSourceColor: (d) => { + const mw = flowOf(d); + if (mw === 0) return FLOW_NONE; + return mw > 0 ? FLOW_FORWARD : FLOW_REVERSE; + }, + getTargetColor: (d) => { + const mw = flowOf(d); + if (mw === 0) return FLOW_NONE; + // brighter head at the receiving end of the flow direction + return mw > 0 + ? [186, 230, 253, 235] + : [254, 215, 170, 235]; + }, + pickable: true, + updateTriggers: { + getWidth: cache.version, + getSourceColor: cache.version, + getTargetColor: cache.version, + }, + }); +} + +// --------------------------------------------------------------------------- +// Vessels +// --------------------------------------------------------------------------- + +const VESSEL_COLOR: Color = [192, 132, 252, 240]; // violet — distinct at sea + +export function buildVesselsLayer( + vessels: EntityRecord[], + cache: StateCache, +): IconLayer { + const data = vessels.filter( + (e) => + pointCoords(e) !== null || + cache.getVesselPosition(e.entity_id) !== undefined, + ); + return new IconLayer({ + id: "vessels", + data, + iconAtlas: getIconAtlas(), + iconMapping: ICON_MAPPING, + getIcon: () => "triangle", + // live position from vessel.position state beats structural geometry + getPosition: (d) => { + const live = cache.getVesselPosition(d.entity_id); + if (live) return [live.lon, live.lat]; + return pointCoords(d) ?? [0, 0]; + }, + // deck angles are CCW degrees; compass heading is CW from north + getAngle: (d) => { + const heading = cache.getVesselPosition(d.entity_id)?.heading_deg; + return typeof heading === "number" ? -heading : 0; + }, + getSize: 20, + sizeUnits: "pixels", + getColor: VESSEL_COLOR, + pickable: true, // hover tooltip labels the vessel by name + updateTriggers: { getPosition: cache.version, getAngle: cache.version }, + }); +} + +// --------------------------------------------------------------------------- +// Observability gaps — first-class visual: pulsing unfilled rings +// --------------------------------------------------------------------------- + +export interface GapMarker { + gap: ObservabilityGapRecord; + entity: EntityRecord; + position: [number, number]; +} + +export function collectGapMarkers( + gaps: ObservabilityGapRecord[], + entityById: Map, +): GapMarker[] { + const markers: GapMarker[] = []; + for (const gap of gaps) { + const entity = entityById.get(gap.entity_id); + if (!entity) continue; + const position = pointCoords(entity); + if (!position) continue; + markers.push({ gap, entity, position }); + } + return markers; +} + +const GAP_PRIORITY_COLOR: Record = { + P1: [248, 113, 113, 235], + P2: [251, 146, 60, 230], + P3: [251, 191, 36, 220], + P4: [148, 163, 184, 200], +}; + +/** + * @param pulse 0..1 phase — radius breathes with it (time-based modulation, + * driven by the animation ticker in main.ts). + */ +export function buildGapsLayer( + markers: GapMarker[], + pulse: number, +): ScatterplotLayer { + const breathe = 1 + 0.4 * Math.sin(pulse * Math.PI * 2); + return new ScatterplotLayer({ + id: "observability-gaps", + data: markers, + getPosition: (d) => d.position, + getRadius: 9_000, + radiusScale: breathe, // uniform-only change per tick — cheap + radiusMinPixels: 6 * breathe, + radiusMaxPixels: 34, + filled: false, // ring style: stroked, unfilled + stroked: true, + getLineColor: (d) => GAP_PRIORITY_COLOR[d.gap.priority] ?? GAP_PRIORITY_COLOR["P4"] ?? GREY, + lineWidthMinPixels: 2, + getLineWidth: 2, + lineWidthUnits: "pixels", + pickable: true, + }); +} + +// --------------------------------------------------------------------------- +// Other facilities — small squares/diamonds coloured by entity_type +// --------------------------------------------------------------------------- + +export const FACILITY_TYPES: ReadonlySet = new Set([ + "Smelter", + "SteelMill", + "Mine", + "CoalMine", + "Refinery", + "LNGPlant", + "ChemicalPlant", + "DataCentre", + "Port", + "Terminal", + "Berth", + "GasProcessingPlant", + "GasField", + "HydroReservoir", + "IndustrialFacility", + "Substation", + "StorageAsset", +]); + +const FACILITY_COLORS: Record = { + Smelter: [251, 113, 133, 235], // rose + SteelMill: [244, 114, 182, 235], + Mine: [217, 119, 6, 235], // ochre + CoalMine: [161, 98, 7, 235], + Refinery: [234, 179, 8, 235], + LNGPlant: [45, 212, 191, 235], // teal + GasProcessingPlant: [20, 184, 166, 235], + GasField: [13, 148, 136, 235], + ChemicalPlant: [163, 230, 53, 235], + DataCentre: [96, 165, 250, 235], + Port: [125, 211, 252, 235], // light blue + Terminal: [56, 189, 248, 235], + Berth: [2, 132, 199, 235], + HydroReservoir: [59, 130, 246, 235], + IndustrialFacility: [216, 180, 254, 235], + Substation: [148, 163, 184, 235], + StorageAsset: [134, 239, 172, 235], +}; + +/** Ports/terminals/berths get diamonds; everything else squares. */ +const DIAMOND_TYPES = new Set(["Port", "Terminal", "Berth"]); + +export function buildFacilitiesLayer( + facilities: EntityRecord[], +): IconLayer { + const data = facilities.filter((e) => pointCoords(e) !== null); + return new IconLayer({ + id: "facilities", + data, + iconAtlas: getIconAtlas(), + iconMapping: ICON_MAPPING, + getIcon: (d) => (DIAMOND_TYPES.has(d.entity_type) ? "diamond" : "square"), + getPosition: (d) => pointCoords(d) ?? [0, 0], + getSize: 13, + sizeUnits: "pixels", + getColor: (d) => FACILITY_COLORS[d.entity_type] ?? [203, 213, 225, 220], + pickable: true, + }); +} diff --git a/intelligence/packages/globe/src/lod.ts b/intelligence/packages/globe/src/lod.ts new file mode 100644 index 0000000..53834be --- /dev/null +++ b/intelligence/packages/globe/src/lod.ts @@ -0,0 +1,94 @@ +// Zoom-gated layer sets. +// +// global z < 2.5 land + grid markers + interconnector arcs +// continental 2.5–4.5 + major generators (>500 MW) +// regional 4.5–8 + all facilities, vessels, gap rings (Gladstone scene) +// facility z > 8 same scene; entity click opens the SDUI side panel + +import type { LayersList } from "@deck.gl/core"; +import { buildLandLayer, buildOceanLayer } from "./basemap.js"; +import { + buildFacilitiesLayer, + buildGapsLayer, + buildGeneratorsLayer, + buildGridMarkersLayer, + buildInterconnectorsLayer, + buildVesselsLayer, + collectGapMarkers, + collectGridMarkers, + FACILITY_TYPES, +} from "./layers.js"; +import type { StateCache } from "./state-cache.js"; +import type { EntityRecord, ObservabilityGapRecord } from "./types.js"; + +export type LodTier = "global" | "continental" | "regional" | "facility"; + +export const MAJOR_GENERATOR_MW = 500; + +export function tierForZoom(zoom: number): LodTier { + if (zoom < 2.5) return "global"; + if (zoom < 4.5) return "continental"; + if (zoom <= 8) return "regional"; + return "facility"; +} + +export interface Scene { + entities: EntityRecord[]; + gaps: ObservabilityGapRecord[]; + entityById: Map; +} + +export function buildScene( + entities: EntityRecord[], + gaps: ObservabilityGapRecord[], +): Scene { + return { + entities, + gaps, + entityById: new Map(entities.map((e) => [e.entity_id, e])), + }; +} + +export function buildLayers( + tier: LodTier, + scene: Scene, + cache: StateCache, + pulse: number, +): LayersList { + const generators = scene.entities.filter( + (e) => e.entity_type === "Generator", + ); + const interconnectors = scene.entities.filter( + (e) => e.entity_type === "Interconnector", + ); + + const layers: LayersList = [ + buildOceanLayer(), + buildLandLayer(), + buildGridMarkersLayer(collectGridMarkers(scene.entities)), + buildInterconnectorsLayer(interconnectors, cache), + ]; + + if (tier === "continental") { + layers.push( + buildGeneratorsLayer(generators, cache, { + minCapacityMw: MAJOR_GENERATOR_MW, + }), + ); + } + + if (tier === "regional" || tier === "facility") { + const vessels = scene.entities.filter((e) => e.entity_type === "Vessel"); + const facilities = scene.entities.filter((e) => + FACILITY_TYPES.has(e.entity_type), + ); + layers.push( + buildGeneratorsLayer(generators, cache), + buildFacilitiesLayer(facilities), + buildVesselsLayer(vessels, cache), + buildGapsLayer(collectGapMarkers(scene.gaps, scene.entityById), pulse), + ); + } + + return layers; +} diff --git a/intelligence/packages/globe/src/main.ts b/intelligence/packages/globe/src/main.ts index 5493888..36e427e 100644 --- a/intelligence/packages/globe/src/main.ts +++ b/intelligence/packages/globe/src/main.ts @@ -1,2 +1,187 @@ -// @pact-tailor/globe — deck.gl globe entry point. Populated in the globe commit. -export {}; +// @pact-tailor/globe — deck.gl _GlobeView entry point. +// GPU-instanced geography; DOM only for the HUD, tooltip and SDUI side panel. + +import "./style.css"; +import { Deck, _GlobeView } from "@deck.gl/core"; +import type { PickingInfo } from "@deck.gl/core"; +import { api, apiStatus, onApiStatusChange } from "./api.js"; +import { Hud } from "./hud.js"; +import { buildLayers, buildScene, tierForZoom } from "./lod.js"; +import type { LodTier, Scene } from "./lod.js"; +import { SidePanel } from "./panel.js"; +import { stateCache } from "./state-cache.js"; +import type { EntityRecord, ObservabilityGapRecord } from "./types.js"; + +const PULSE_MS = 1600; // gap-ring breathing period +const TICK_MS = 90; // pulse animation cadence (~11 fps, uniform-only updates) +const RELOAD_MS = 30_000; // structural data retry when API was absent + +const globeEl = document.getElementById("globe"); +if (!globeEl) throw new Error("missing #globe container"); + +const hud = new Hud(document.body); +const panel = new SidePanel(document.body); + +let scene: Scene = buildScene([], []); +let tier: LodTier = tierForZoom(3.2); +hud.setTier(tier); +hud.setEntityCount(0); + +// -- picking helpers --------------------------------------------------------- + +function pickedEntity(obj: unknown): EntityRecord | null { + if (!obj || typeof obj !== "object") return null; + const rec = obj as Record; + if (typeof rec["entity_id"] === "string" && typeof rec["entity_type"] === "string") { + return obj as EntityRecord; + } + // wrapper data (interconnector arcs, gap markers, grid markers) + const inner = rec["entity"]; + if (inner && typeof inner === "object") return inner as EntityRecord; + return null; +} + +function pickLabel(obj: unknown): string | null { + if (!obj || typeof obj !== "object") return null; + const rec = obj as Record; + const gap = rec["gap"] as ObservabilityGapRecord | undefined; + if (gap && typeof gap === "object") { + const entity = pickedEntity(obj); + return `⚠ gap: ${gap.desired_metric} (${gap.priority})${entity ? `\n${entity.name}` : ""}`; + } + const entity = pickedEntity(obj); + if (entity) { + const flow = stateCache.findMegawatts(entity.entity_id); + const flowLine = + entity.entity_type === "Interconnector" && flow !== undefined + ? `\n${flow.toFixed(0)} MW` + : ""; + return `${entity.name}\n${entity.entity_type}${flowLine}`; + } + if (typeof rec["name"] === "string") return rec["name"]; + return null; +} + +function handleClick(info: PickingInfo): void { + const entity = pickedEntity(info.object); + if (entity) { + // facility LOD by interaction: entity click opens the SDUI side panel + void panel.open(entity.entity_id, entity); + } else if (panel.isOpen) { + panel.close(); + } +} + +// -- deck -------------------------------------------------------------------- + +const deck = new Deck({ + parent: globeEl as HTMLDivElement, + // resolution: degrees per mesh subdivision when draping flat geometry on + // the sphere — 5° keeps the coarse basemap looking curved, still cheap + views: new _GlobeView({ resolution: 5 }), + controller: true, + initialViewState: { + longitude: 146, + latitude: -24, + zoom: 3.2, + minZoom: 0.4, + maxZoom: 13, + }, + layers: [], + onViewStateChange: ({ viewState }) => { + const zoom = (viewState as { zoom?: number }).zoom; + if (typeof zoom !== "number") return; + const next = tierForZoom(zoom); + if (next !== tier) { + tier = next; + hud.setTier(tier); + render(); + } + }, + onClick: handleClick, + getTooltip: (info: PickingInfo) => { + const label = pickLabel(info.object); + return label ? { text: label } : null; + }, +}); + +function render(): void { + const pulse = (Date.now() % PULSE_MS) / PULSE_MS; + deck.setProps({ layers: buildLayers(tier, scene, stateCache, pulse) }); +} + +// -- data -------------------------------------------------------------------- + +async function loadData(): Promise { + const [gridsRes, entitiesRes, gapsRes] = await Promise.all([ + api.getGrids(), + api.getEntities(), + api.getGaps(), + ]); + const entities = entitiesRes?.entities ?? []; + const gaps = gapsRes?.gaps ?? []; + scene = buildScene(entities, gaps); + hud.setEntityCount(entities.length); + updateHudConnection(); + render(); + + // Seed interconnector flows from grid summaries so arcs carry direction + // and width on first paint, before the first SSE tick lands. + const grids = gridsRes?.grids ?? []; + await Promise.all( + grids.slice(0, 4).map(async (g) => { + const summary = await api.getGridSummary(g.grid_id); + for (const flow of summary?.live?.flows ?? []) { + if (typeof flow.mw === "number") { + stateCache.seed(flow.entity_id, "flow_mw", flow.mw, "MW", flow.event_time); + } + } + }), + ); +} + +// -- status wiring ----------------------------------------------------------- + +function updateHudConnection(): void { + if (apiStatus.down) { + hud.setConnection("offline", "API offline — structural view"); + return; + } + switch (stateCache.connection) { + case "live": + hud.setConnection("live", "live SSE — replayed fixtures"); + break; + case "connecting": + hud.setConnection("connecting", "connecting…"); + break; + case "offline": + hud.setConnection("offline", "stream offline"); + break; + } +} + +onApiStatusChange(updateHudConnection); +stateCache.onConnectionChange(updateHudConnection); +// cache throttles change notifications to at most 4 fps +stateCache.onChange(render); + +// gap-ring pulse: uniform-only radius modulation, active when rings visible +setInterval(() => { + if ((tier === "regional" || tier === "facility") && scene.gaps.length > 0) { + render(); + } +}, TICK_MS); + +// retry structural load while the API is absent or nothing has arrived yet +setInterval(() => { + if (apiStatus.down || scene.entities.length === 0) void loadData(); +}, RELOAD_MS); + +// -- boot -------------------------------------------------------------------- + +stateCache.start(); +updateHudConnection(); +render(); +void loadData(); + +export { deck }; diff --git a/intelligence/packages/globe/src/panel.ts b/intelligence/packages/globe/src/panel.ts new file mode 100644 index 0000000..a9355fc --- /dev/null +++ b/intelligence/packages/globe/src/panel.ts @@ -0,0 +1,515 @@ +// SDUI side panel — plain TS + CSS, no framework. +// +// On entity click we fetch /api/intel/sdui/{id} and render the server-composed +// layout. SDUI rule: every panel fetches its own data.endpoint — panels never +// receive inline values. Honesty rule: whenever data is present, a "replayed +// fixture data — not live" banner is shown (the API replays fixtures here). + +import { api, apiStatus } from "./api.js"; +import type { + EntityRecord, + EntityStateResponse, + InferenceRecord, + ObservabilityGapRecord, + SduiPanel, + SduiPanelDoc, + StateEntry, +} from "./types.js"; + +const SVG_NS = "http://www.w3.org/2000/svg"; + +function el( + tag: string, + className?: string, + text?: string, +): HTMLElement { + const node = document.createElement(tag); + if (className) node.className = className; + if (text !== undefined) node.textContent = text; + return node; +} + +function asRecord(v: unknown): Record | null { + return v !== null && typeof v === "object" && !Array.isArray(v) + ? (v as Record) + : null; +} + +/** Liberal array extraction: bare array, or first array under known keys. */ +function asArray(json: unknown, keys: string[]): unknown[] { + if (Array.isArray(json)) return json; + const rec = asRecord(json); + if (!rec) return []; + for (const k of keys) { + const v = rec[k]; + if (Array.isArray(v)) return v; + } + return []; +} + +function fmtValue(v: unknown): string { + if (typeof v === "number") { + const abs = Math.abs(v); + if (abs >= 1000) return v.toLocaleString(undefined, { maximumFractionDigits: 0 }); + if (abs >= 10) return v.toLocaleString(undefined, { maximumFractionDigits: 1 }); + return v.toLocaleString(undefined, { maximumFractionDigits: 2 }); + } + if (typeof v === "string") return v; + const rec = asRecord(v); + if (rec && typeof rec.lat === "number" && typeof rec.lon === "number") { + return `${rec.lat.toFixed(3)}, ${rec.lon.toFixed(3)}`; + } + return v === null || v === undefined ? "—" : JSON.stringify(v); +} + +function fmtTime(iso: unknown): string { + if (typeof iso !== "string") return ""; + const d = new Date(iso); + return Number.isNaN(d.getTime()) ? iso : d.toISOString().replace("T", " ").slice(0, 16) + "Z"; +} + +export class SidePanel { + private root: HTMLElement; + private banner: HTMLElement; + private header: HTMLElement; + private body: HTMLElement; + private openToken = 0; + + constructor(parent: HTMLElement) { + this.root = el("aside", "panel panel-hidden"); + const bar = el("div", "panel-bar"); + const close = el("button", "panel-close", "×"); + close.setAttribute("aria-label", "Close panel"); + close.addEventListener("click", () => this.close()); + bar.appendChild(close); + this.banner = el("div", "panel-banner", "replayed fixture data — not live"); + this.banner.hidden = true; + this.header = el("div", "panel-header"); + this.body = el("div", "panel-body"); + this.root.appendChild(bar); + this.root.appendChild(this.banner); + this.root.appendChild(this.header); + this.root.appendChild(this.body); + parent.appendChild(this.root); + } + + get isOpen(): boolean { + return !this.root.classList.contains("panel-hidden"); + } + + close(): void { + this.openToken += 1; + this.root.classList.add("panel-hidden"); + } + + async open(entityId: string, entity?: EntityRecord): Promise { + const token = ++this.openToken; + this.root.classList.remove("panel-hidden"); + this.banner.hidden = true; + this.header.replaceChildren(); + this.body.replaceChildren(el("div", "panel-loading", "composing panel…")); + + const doc = await api.getSdui(entityId); + if (token !== this.openToken) return; // superseded by a newer open/close + + this.renderHeader(entityId, entity, doc); + this.body.replaceChildren(); + + if (!doc) { + const msg = apiStatus.down + ? "API offline — structural view only. No SDUI document available." + : "No SDUI document for this entity."; + this.body.appendChild(el("div", "panel-empty", msg)); + return; + } + + for (const panel of doc.layout) { + const section = el("section", `panel-section panel-kind-${panel.panel}`); + section.appendChild(el("h3", "panel-section-title", titleFor(panel))); + this.body.appendChild(section); + void this.fillSection(token, section, panel, doc); + } + + this.renderSplatMount(entity); + } + + // -- header --------------------------------------------------------------- + + private renderHeader( + entityId: string, + entity: EntityRecord | undefined, + doc: SduiPanelDoc | null, + ): void { + this.header.replaceChildren(); + this.header.appendChild( + el("h2", "panel-name", entity?.name ?? doc?.entity_id ?? entityId), + ); + const meta = el("div", "panel-meta"); + const type = doc?.entity_type ?? entity?.entity_type; + if (type) meta.appendChild(el("span", "chip chip-type", type)); + if (entity?.observability) { + meta.appendChild( + el( + "span", + `chip chip-obs obs-${entity.observability.toLowerCase()}`, + entity.observability, + ), + ); + } + for (const s of doc?.situation ?? []) { + meta.appendChild(el("span", "chip chip-situation", s)); + } + this.header.appendChild(meta); + this.header.appendChild(el("div", "panel-id", entityId)); + } + + // -- section data --------------------------------------------------------- + + private async fillSection( + token: number, + section: HTMLElement, + panel: SduiPanel, + doc: SduiPanelDoc, + ): Promise { + const endpoint = panel.data?.endpoint; + const bodyEl = el("div", "panel-section-body"); + section.appendChild(bodyEl); + + if (!endpoint) { + // SDUI rule: no endpoint, no data — we never render inline values. + bodyEl.appendChild( + el("div", "panel-empty", "no data endpoint declared"), + ); + return; + } + + bodyEl.appendChild(el("div", "panel-loading", "fetching…")); + const json = await api.fetchEndpoint(endpoint); + if (token !== this.openToken) return; + bodyEl.replaceChildren(); + + if (json === null) { + bodyEl.appendChild( + el( + "div", + "panel-empty", + apiStatus.down ? "API offline" : "no data from endpoint", + ), + ); + return; + } + + this.banner.hidden = false; // data present → honesty banner on + + switch (panel.panel) { + case "headline-state": + renderHeadlineState(bodyEl, json); + break; + case "timeseries": + renderTimeseries(bodyEl, json, panel); + break; + case "gap-card": + renderGapCard(bodyEl, json); + break; + case "graph-neighborhood": + renderGraphNeighborhood(bodyEl, json, doc.entity_id); + break; + case "inference-list": + renderInferenceList(bodyEl, json); + break; + default: + bodyEl.appendChild( + el("div", "panel-empty", `unsupported panel type: ${panel.panel}`), + ); + } + } + + /** 3DGS mount contract: properties.visualisation.splat_url, when present. */ + private renderSplatMount(entity: EntityRecord | undefined): void { + const vis = asRecord(entity?.properties?.["visualisation"]); + const splatUrl = vis?.["splat_url"]; + if (typeof splatUrl !== "string" || !splatUrl) return; + const section = el("section", "panel-section panel-kind-splat"); + section.appendChild(el("h3", "panel-section-title", "site view")); + section.appendChild( + el( + "div", + "panel-empty", + "3D Gaussian Splatting site view reserved (out of scope in this workspace)", + ), + ); + this.body.appendChild(section); + } +} + +// --------------------------------------------------------------------------- +// Panel renderers +// --------------------------------------------------------------------------- + +function titleFor(panel: SduiPanel): string { + const names: Record = { + "headline-state": "live state", + timeseries: "timeseries", + "gap-card": "observability gaps", + "graph-neighborhood": "neighborhood", + "inference-list": "inferences", + }; + return names[panel.panel] ?? panel.panel; +} + +function renderHeadlineState(parent: HTMLElement, json: unknown): void { + const states = asArray(json, ["states"]) as StateEntry[]; + if (states.length === 0) { + parent.appendChild(el("div", "panel-empty", "no live state")); + return; + } + const grid = el("div", "headline-grid"); + for (const s of states) { + if (!asRecord(s)) continue; + const card = el("div", "headline-card"); + const value = el("div", "headline-value", fmtValue(s.value)); + if (s.unit) value.appendChild(el("span", "headline-unit", ` ${s.unit}`)); + card.appendChild(value); + card.appendChild(el("div", "headline-metric", s.metric ?? "")); + const foot = el("div", "headline-foot", fmtTime(s.event_time)); + if (s.quality && s.quality !== "good") { + foot.appendChild(el("span", `chip chip-quality q-${s.quality}`, s.quality)); + } + card.appendChild(foot); + grid.appendChild(card); + } + parent.appendChild(grid); +} + +function renderTimeseries( + parent: HTMLElement, + json: unknown, + panel: SduiPanel, +): void { + const rows = asArray(json, ["observations", "states", "items", "data"]); + const points: Array<{ t: number; v: number }> = []; + for (const row of rows) { + const rec = asRecord(row); + if (!rec) continue; + const v = rec["value"]; + const t = Date.parse(String(rec["event_time"] ?? "")); + if (typeof v === "number" && Number.isFinite(v) && !Number.isNaN(t)) { + points.push({ t, v }); + } + } + points.sort((a, b) => a.t - b.t); + + const metric = typeof panel.props?.["metric"] === "string" ? String(panel.props["metric"]) : ""; + const windowLabel = typeof panel.props?.["window"] === "string" ? String(panel.props["window"]) : ""; + if (metric || windowLabel) { + parent.appendChild( + el("div", "ts-caption", [metric, windowLabel].filter(Boolean).join(" · ")), + ); + } + + if (points.length < 2) { + parent.appendChild(el("div", "panel-empty", "not enough observations")); + return; + } + + const W = 264; + const H = 64; + const PAD = 4; + let min = Infinity; + let max = -Infinity; + for (const p of points) { + if (p.v < min) min = p.v; + if (p.v > max) max = p.v; + } + const span = max - min || 1; + const t0 = points[0]?.t ?? 0; + const t1 = points[points.length - 1]?.t ?? 1; + const tSpan = t1 - t0 || 1; + + const svg = document.createElementNS(SVG_NS, "svg"); + svg.setAttribute("viewBox", `0 0 ${W} ${H}`); + svg.setAttribute("class", "sparkline"); + const poly = document.createElementNS(SVG_NS, "polyline"); + poly.setAttribute( + "points", + points + .map((p) => { + const x = PAD + ((p.t - t0) / tSpan) * (W - PAD * 2); + const y = H - PAD - ((p.v - min) / span) * (H - PAD * 2); + return `${x.toFixed(1)},${y.toFixed(1)}`; + }) + .join(" "), + ); + poly.setAttribute("fill", "none"); + poly.setAttribute("stroke", "currentColor"); + poly.setAttribute("stroke-width", "1.5"); + svg.appendChild(poly); + parent.appendChild(svg); + + const last = points[points.length - 1]; + parent.appendChild( + el( + "div", + "ts-range", + `min ${fmtValue(min)} · max ${fmtValue(max)} · latest ${fmtValue(last?.v)}`, + ), + ); +} + +function renderGapCard(parent: HTMLElement, json: unknown): void { + const gaps = asArray(json, ["gaps"]) as ObservabilityGapRecord[]; + if (gaps.length === 0) { + parent.appendChild(el("div", "panel-empty", "no open gaps")); + return; + } + for (const gap of gaps) { + if (!asRecord(gap)) continue; + const card = el("div", "gap-card"); + const head = el("div", "gap-head"); + head.appendChild(el("span", `chip chip-priority p-${(gap.priority ?? "P4").toLowerCase()}`, gap.priority ?? "P?")); + head.appendChild(el("span", "gap-metric", gap.desired_metric ?? "")); + head.appendChild(el("span", "chip chip-status", gap.status ?? "")); + card.appendChild(head); + const vals = el("div", "gap-values"); + if (gap.commercial_value) vals.appendChild(el("span", "gap-val", `commercial: ${gap.commercial_value}`)); + if (gap.strategic_value) vals.appendChild(el("span", "gap-val", `strategic: ${gap.strategic_value}`)); + card.appendChild(vals); + const opts = gap.instrumentation_options ?? []; + if (opts.length > 0) { + const list = el("ul", "gap-options"); + for (const opt of opts) { + const item = el("li", "gap-option", opt.kind); + if (opt.indicative_cost_band) { + item.appendChild(el("span", "gap-cost", ` · cost ${opt.indicative_cost_band}`)); + } + if (opt.notes) item.appendChild(el("span", "gap-note", ` — ${opt.notes}`)); + list.appendChild(item); + } + card.appendChild(list); + } + if (gap.best_available_proxy) { + card.appendChild( + el( + "div", + "gap-proxy", + `proxy: ${gap.best_available_proxy.metric} (${gap.best_available_proxy.source_id})`, + ), + ); + } + parent.appendChild(card); + } +} + +function nameOf(v: unknown): string | null { + if (typeof v === "string") return v; + const rec = asRecord(v); + if (!rec) return null; + if (typeof rec["name"] === "string") return rec["name"]; + if (typeof rec["entity_id"] === "string") return rec["entity_id"]; + return null; +} + +function renderNameList(parent: HTMLElement, label: string, items: string[]): void { + if (items.length === 0) return; + parent.appendChild(el("h4", "graph-label", label)); + const list = el("ul", "graph-list"); + for (const name of items) list.appendChild(el("li", "graph-item", name)); + parent.appendChild(list); +} + +function renderGraphNeighborhood( + parent: HTMLElement, + json: unknown, + entityId: string, +): void { + const rec = asRecord(json) ?? {}; + + // Shape 1: explicit upstream/downstream lists + const upstream = asArray(rec["upstream"], []).map(nameOf).filter((n): n is string => n !== null); + const downstream = asArray(rec["downstream"], []).map(nameOf).filter((n): n is string => n !== null); + if (upstream.length > 0 || downstream.length > 0) { + renderNameList(parent, "upstream", upstream); + renderNameList(parent, "downstream", downstream); + return; + } + + // Shape 2: nodes + edges/relationships — derive direction relative to entity + const nodes = asArray(rec["nodes"], []) as Array>; + const edges = asArray(rec["edges"] ?? rec["relationships"], []) as Array>; + if (nodes.length > 0 && edges.length > 0) { + const nameById = new Map(); + for (const n of nodes) { + const id = typeof n["entity_id"] === "string" ? n["entity_id"] : null; + if (id) nameById.set(id, typeof n["name"] === "string" ? n["name"] : id); + } + const up: string[] = []; + const down: string[] = []; + for (const e of edges) { + const from = typeof e["from_id"] === "string" ? e["from_id"] : null; + const to = typeof e["to_id"] === "string" ? e["to_id"] : null; + if (to === entityId && from) up.push(nameById.get(from) ?? from); + else if (from === entityId && to) down.push(nameById.get(to) ?? to); + } + renderNameList(parent, "upstream", up); + renderNameList(parent, "downstream", down); + if (up.length === 0 && down.length === 0) { + renderNameList(parent, "neighbors", [...nameById.values()].filter((n) => n !== entityId)); + } + return; + } + + // Shape 3: flat neighbor list + const neighbors = asArray(json, ["neighbors", "entities"]).map(nameOf).filter((n): n is string => n !== null); + if (neighbors.length > 0) { + renderNameList(parent, "neighbors", neighbors); + return; + } + parent.appendChild(el("div", "panel-empty", "no neighborhood data")); +} + +function renderInferenceList(parent: HTMLElement, json: unknown): void { + const inferences = asArray(json, ["inferences", "items"]) as InferenceRecord[]; + if (inferences.length === 0) { + parent.appendChild(el("div", "panel-empty", "no inferences")); + return; + } + for (const inf of inferences) { + if (!asRecord(inf)) continue; + const status = inf.status ?? "INFERRED"; + const card = el("div", `inference-card status-${status.toLowerCase()}`); + const head = el("div", "inference-head"); + head.appendChild(el("span", `chip chip-inf-status s-${status.toLowerCase()}`, status)); + if (inf.tier) head.appendChild(el("span", "chip chip-tier", `tier ${inf.tier}`)); + card.appendChild(head); + card.appendChild(el("div", "inference-claim", inf.claim ?? "")); + + const conf = typeof inf.confidence === "number" ? Math.max(0, Math.min(1, inf.confidence)) : null; + if (conf !== null) { + const bar = el("div", "confidence-bar"); + const fill = el("div", "confidence-fill"); + fill.style.width = `${Math.round(conf * 100)}%`; + bar.appendChild(fill); + const wrap = el("div", "confidence-wrap"); + wrap.appendChild(bar); + wrap.appendChild(el("span", "confidence-label", `${Math.round(conf * 100)}%`)); + card.appendChild(wrap); + } + if (inf.method) card.appendChild(el("div", "inference-method", inf.method)); + + const contrary = inf.contrary_evidence ?? []; + if (contrary.length > 0) { + // Honesty rule: contrary evidence is visibly marked, never buried. + const block = el("div", "contrary-evidence"); + block.appendChild( + el("div", "contrary-title", `⚠ contrary evidence (${contrary.length})`), + ); + const list = el("ul", "contrary-list"); + for (const ev of contrary) { + list.appendChild(el("li", "contrary-item", `${ev.kind}: ${ev.ref}`)); + } + block.appendChild(list); + card.appendChild(block); + } + parent.appendChild(card); + } +} diff --git a/intelligence/packages/globe/src/state-cache.ts b/intelligence/packages/globe/src/state-cache.ts new file mode 100644 index 0000000..865d024 --- /dev/null +++ b/intelligence/packages/globe/src/state-cache.ts @@ -0,0 +1,227 @@ +// Live state cache fed by the /api/intel/stream SSE feed. +// Map>; deck re-renders are +// triggered through onChange, throttled to at most 4 fps. Reconnects with +// exponential backoff. When the API is absent the cache simply stays empty +// and the HUD reports "API offline — structural view". + +import { API_BASE } from "./api.js"; +import type { StreamStateEvent, VesselPositionValue } from "./types.js"; + +export type ConnectionState = "connecting" | "live" | "offline"; + +export interface CachedMetric { + value: number | string | Record; + unit?: string; + event_time: string; +} + +const THROTTLE_MS = 250; // 4 fps max +const BACKOFF_MIN_MS = 1000; +const BACKOFF_MAX_MS = 30_000; + +export class StateCache { + private byEntity = new Map>(); + private source: EventSource | null = null; + private backoffMs = BACKOFF_MIN_MS; + private reconnectTimer: ReturnType | null = null; + private throttleTimer: ReturnType | null = null; + private dirty = false; + private changeListeners = new Set<() => void>(); + private connectionListeners = new Set<(s: ConnectionState) => void>(); + + /** Monotonic version bump per applied update — used for deck updateTriggers. */ + version = 0; + connection: ConnectionState = "connecting"; + inferenceCount = 0; + + start(): void { + if (this.source) return; + this.connect(); + } + + stop(): void { + if (this.reconnectTimer !== null) clearTimeout(this.reconnectTimer); + this.reconnectTimer = null; + this.source?.close(); + this.source = null; + } + + get(entityId: string, metric: string): CachedMetric | undefined { + return this.byEntity.get(entityId)?.get(metric); + } + + metricsFor(entityId: string): ReadonlyMap | undefined { + return this.byEntity.get(entityId); + } + + getNumber(entityId: string, metric: string): number | undefined { + const v = this.get(entityId, metric)?.value; + return typeof v === "number" && Number.isFinite(v) ? v : undefined; + } + + /** First numeric hit among candidate metric names. */ + getFirstNumber(entityId: string, metrics: string[]): number | undefined { + for (const m of metrics) { + const v = this.getNumber(entityId, m); + if (v !== undefined) return v; + } + return undefined; + } + + /** + * Fallback lookup when the exact metric name is unknown: first numeric + * metric whose unit is MW or whose name ends in the given suffix. + */ + findMegawatts(entityId: string): number | undefined { + const metrics = this.byEntity.get(entityId); + if (!metrics) return undefined; + for (const [name, entry] of metrics) { + if (typeof entry.value !== "number" || !Number.isFinite(entry.value)) { + continue; + } + if (entry.unit === "MW" || name.endsWith("_mw") || name.endsWith(".mw")) { + return entry.value; + } + } + return undefined; + } + + getVesselPosition(entityId: string): VesselPositionValue | undefined { + const entry = + this.get(entityId, "vessel.position") ?? this.get(entityId, "position"); + const v = entry?.value; + if ( + v !== null && + typeof v === "object" && + typeof (v as Record).lat === "number" && + typeof (v as Record).lon === "number" + ) { + return v as unknown as VesselPositionValue; + } + return undefined; + } + + /** + * Seed a metric from a REST snapshot (e.g. grid summary flows) — only when + * no fresher SSE value already occupies the slot. + */ + seed( + entityId: string, + metric: string, + value: number | string | Record, + unit: string | undefined, + eventTime: string, + ): void { + if (this.get(entityId, metric)) return; + let metrics = this.byEntity.get(entityId); + if (!metrics) { + metrics = new Map(); + this.byEntity.set(entityId, metrics); + } + const entry: CachedMetric = { value, event_time: eventTime }; + if (unit !== undefined) entry.unit = unit; + metrics.set(metric, entry); + this.markDirty(); + } + + onChange(fn: () => void): () => void { + this.changeListeners.add(fn); + return () => this.changeListeners.delete(fn); + } + + onConnectionChange(fn: (s: ConnectionState) => void): () => void { + this.connectionListeners.add(fn); + return () => this.connectionListeners.delete(fn); + } + + // -- internals ------------------------------------------------------------ + + private connect(): void { + this.setConnection(this.connection === "live" ? "live" : "connecting"); + let source: EventSource; + try { + source = new EventSource(`${API_BASE}/stream`); + } catch { + this.scheduleReconnect(); + return; + } + this.source = source; + + source.onopen = () => { + this.backoffMs = BACKOFF_MIN_MS; + this.setConnection("live"); + }; + + source.onerror = () => { + // EventSource auto-retries, but when the API is down entirely we take + // over with our own capped exponential backoff. + source.close(); + if (this.source === source) this.source = null; + this.setConnection("offline"); + this.scheduleReconnect(); + }; + + const onState = (ev: Event) => { + this.applyStateEvent((ev as MessageEvent).data); + }; + source.addEventListener("state.updated", onState); + source.addEventListener("state.corrected", onState); + source.addEventListener("inference", () => { + this.inferenceCount += 1; + this.markDirty(); + }); + } + + private scheduleReconnect(): void { + if (this.reconnectTimer !== null) return; + const delay = this.backoffMs; + this.backoffMs = Math.min(this.backoffMs * 2, BACKOFF_MAX_MS); + this.reconnectTimer = setTimeout(() => { + this.reconnectTimer = null; + this.connect(); + }, delay); + } + + private applyStateEvent(raw: string): void { + let parsed: StreamStateEvent; + try { + parsed = JSON.parse(raw) as StreamStateEvent; + } catch { + return; + } + const state = parsed?.state; + if (!state || typeof state.entity_id !== "string" || !state.metric) return; + let metrics = this.byEntity.get(state.entity_id); + if (!metrics) { + metrics = new Map(); + this.byEntity.set(state.entity_id, metrics); + } + const entry: CachedMetric = { + value: state.value, + event_time: state.event_time, + }; + if (state.unit !== undefined) entry.unit = state.unit; + metrics.set(state.metric, entry); + this.markDirty(); + } + + private markDirty(): void { + this.version += 1; + this.dirty = true; + if (this.throttleTimer !== null) return; + this.throttleTimer = setTimeout(() => { + this.throttleTimer = null; + if (!this.dirty) return; + this.dirty = false; + for (const fn of this.changeListeners) fn(); + }, THROTTLE_MS); + } + + private setConnection(s: ConnectionState): void { + if (this.connection === s) return; + this.connection = s; + for (const fn of this.connectionListeners) fn(s); + } +} + +export const stateCache = new StateCache(); diff --git a/intelligence/packages/globe/src/style.css b/intelligence/packages/globe/src/style.css new file mode 100644 index 0000000..5efaddf --- /dev/null +++ b/intelligence/packages/globe/src/style.css @@ -0,0 +1,524 @@ +/* PACT.TAILOR globe — dark theme, system fonts, crisp. */ + +:root { + --bg: #030712; + --bg-raised: rgba(9, 14, 28, 0.92); + --border: rgba(71, 85, 105, 0.45); + --text: #e2e8f0; + --text-dim: #94a3b8; + --text-faint: #64748b; + --accent: #38bdf8; + --live: #4ade80; + --warn: #fbbf24; + --danger: #f87171; + --mono: ui-monospace, "SF Mono", SFMono-Regular, Menlo, Consolas, monospace; +} + +* { + box-sizing: border-box; +} + +html, +body { + margin: 0; + padding: 0; + height: 100%; + overflow: hidden; + background: + radial-gradient(ellipse 120% 90% at 30% 20%, #0b1226 0%, var(--bg) 65%); + color: var(--text); + font-family: + -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", + Arial, sans-serif; + font-size: 14px; + -webkit-font-smoothing: antialiased; +} + +#globe { + position: fixed; + inset: 0; +} + +#globe canvas { + outline: none; +} + +/* ------------------------------------------------------------------ HUD -- */ + +.hud { + position: fixed; + top: 16px; + left: 16px; + z-index: 10; + padding: 12px 16px; + background: var(--bg-raised); + border: 1px solid var(--border); + border-radius: 10px; + backdrop-filter: blur(6px); + max-width: 420px; + pointer-events: none; + user-select: none; +} + +.hud-title { + font-size: 13px; + font-weight: 600; + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--text); +} + +.hud-row { + display: flex; + gap: 8px; + margin-top: 8px; + flex-wrap: wrap; +} + +.hud-chip { + font-family: var(--mono); + font-size: 11px; + padding: 2px 8px; + border-radius: 999px; + border: 1px solid var(--border); + color: var(--text-dim); + white-space: nowrap; +} + +.hud-tier { + color: var(--accent); + border-color: rgba(56, 189, 248, 0.4); +} + +.hud-conn.is-live { + color: var(--live); + border-color: rgba(74, 222, 128, 0.4); +} + +.hud-conn.is-offline { + color: var(--danger); + border-color: rgba(248, 113, 113, 0.4); +} + +.hud-conn.is-connecting { + color: var(--warn); + border-color: rgba(251, 191, 36, 0.4); +} + +/* ---------------------------------------------------------------- panel -- */ + +.panel { + position: fixed; + top: 0; + right: 0; + bottom: 0; + z-index: 20; + width: 340px; + max-width: 92vw; + overflow-y: auto; + background: var(--bg-raised); + border-left: 1px solid var(--border); + backdrop-filter: blur(8px); + transform: translateX(0); + transition: transform 0.18s ease-out; +} + +.panel-hidden { + transform: translateX(105%); + pointer-events: none; +} + +.panel-bar { + display: flex; + justify-content: flex-end; + padding: 8px 8px 0; +} + +.panel-close { + appearance: none; + background: transparent; + border: 1px solid var(--border); + border-radius: 6px; + color: var(--text-dim); + font-size: 16px; + line-height: 1; + padding: 3px 9px; + cursor: pointer; +} + +.panel-close:hover { + color: var(--text); + border-color: var(--text-faint); +} + +.panel-banner { + margin: 8px 14px 0; + padding: 6px 10px; + font-size: 11px; + font-family: var(--mono); + letter-spacing: 0.04em; + color: var(--warn); + background: rgba(251, 191, 36, 0.08); + border: 1px solid rgba(251, 191, 36, 0.35); + border-radius: 6px; + text-align: center; +} + +.panel-header { + padding: 10px 16px 4px; +} + +.panel-name { + margin: 0; + font-size: 17px; + font-weight: 650; +} + +.panel-meta { + display: flex; + gap: 6px; + flex-wrap: wrap; + margin-top: 6px; +} + +.panel-id { + margin-top: 6px; + font-family: var(--mono); + font-size: 10.5px; + color: var(--text-faint); + word-break: break-all; +} + +.panel-body { + padding: 6px 16px 24px; +} + +.panel-section { + margin-top: 16px; + border-top: 1px solid var(--border); + padding-top: 10px; +} + +.panel-section-title { + margin: 0 0 8px; + font-size: 11px; + font-weight: 600; + letter-spacing: 0.1em; + text-transform: uppercase; + color: var(--text-faint); +} + +.panel-loading, +.panel-empty { + font-size: 12.5px; + color: var(--text-faint); + font-style: italic; + padding: 4px 0; +} + +/* chips */ + +.chip { + display: inline-block; + font-family: var(--mono); + font-size: 10.5px; + padding: 1.5px 7px; + border-radius: 999px; + border: 1px solid var(--border); + color: var(--text-dim); +} + +.chip-type { + color: var(--accent); + border-color: rgba(56, 189, 248, 0.4); +} + +.chip-situation { + color: var(--warn); + border-color: rgba(251, 191, 36, 0.35); +} + +.chip-priority.p-p1 { + color: var(--danger); + border-color: rgba(248, 113, 113, 0.5); +} + +.chip-priority.p-p2 { + color: #fb923c; + border-color: rgba(251, 146, 60, 0.5); +} + +.chip-priority.p-p3 { + color: var(--warn); + border-color: rgba(251, 191, 36, 0.5); +} + +.chip-quality { + margin-left: 6px; + color: var(--warn); +} + +/* headline-state */ + +.headline-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 8px; +} + +.headline-card { + background: rgba(15, 23, 42, 0.6); + border: 1px solid var(--border); + border-radius: 8px; + padding: 8px 10px; +} + +.headline-value { + font-size: 20px; + font-weight: 650; + font-variant-numeric: tabular-nums; +} + +.headline-unit { + font-size: 11px; + font-weight: 500; + color: var(--text-dim); +} + +.headline-metric { + margin-top: 2px; + font-family: var(--mono); + font-size: 10.5px; + color: var(--text-dim); + word-break: break-all; +} + +.headline-foot { + margin-top: 4px; + font-size: 10px; + color: var(--text-faint); + font-family: var(--mono); +} + +/* timeseries */ + +.ts-caption { + font-family: var(--mono); + font-size: 11px; + color: var(--text-dim); + margin-bottom: 4px; +} + +.sparkline { + width: 100%; + height: 64px; + color: var(--accent); + background: rgba(15, 23, 42, 0.6); + border: 1px solid var(--border); + border-radius: 8px; +} + +.ts-range { + margin-top: 4px; + font-family: var(--mono); + font-size: 10.5px; + color: var(--text-faint); +} + +/* gap-card */ + +.gap-card { + background: rgba(15, 23, 42, 0.6); + border: 1px solid rgba(251, 146, 60, 0.3); + border-radius: 8px; + padding: 8px 10px; + margin-bottom: 8px; +} + +.gap-head { + display: flex; + align-items: center; + gap: 7px; + flex-wrap: wrap; +} + +.gap-metric { + font-family: var(--mono); + font-size: 12px; + color: var(--text); +} + +.gap-values { + display: flex; + gap: 10px; + margin-top: 5px; + font-size: 11px; + color: var(--text-dim); +} + +.gap-options { + margin: 6px 0 0; + padding-left: 18px; + font-size: 11.5px; + color: var(--text-dim); +} + +.gap-option { + margin-bottom: 2px; + font-family: var(--mono); +} + +.gap-cost, +.gap-note { + color: var(--text-faint); + font-family: + -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; +} + +.gap-proxy { + margin-top: 6px; + font-size: 11px; + font-family: var(--mono); + color: var(--text-faint); +} + +/* graph-neighborhood */ + +.graph-label { + margin: 8px 0 4px; + font-size: 10.5px; + font-weight: 600; + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--text-faint); +} + +.graph-list { + margin: 0; + padding-left: 18px; + font-size: 12.5px; + color: var(--text-dim); +} + +.graph-item { + margin-bottom: 2px; +} + +/* inference-list */ + +.inference-card { + background: rgba(15, 23, 42, 0.6); + border: 1px solid var(--border); + border-radius: 8px; + padding: 8px 10px; + margin-bottom: 8px; +} + +.inference-card.status-contested { + border-color: rgba(248, 113, 113, 0.45); +} + +.inference-head { + display: flex; + gap: 6px; + align-items: center; +} + +.chip-inf-status.s-inferred { + color: var(--accent); + border-color: rgba(56, 189, 248, 0.4); +} + +.chip-inf-status.s-corroborated { + color: var(--live); + border-color: rgba(74, 222, 128, 0.45); +} + +.chip-inf-status.s-contested { + color: var(--danger); + border-color: rgba(248, 113, 113, 0.5); +} + +.chip-inf-status.s-verified { + color: var(--live); + border-color: rgba(74, 222, 128, 0.6); +} + +.chip-inf-status.s-retracted { + color: var(--text-faint); + text-decoration: line-through; +} + +.inference-claim { + margin-top: 6px; + font-size: 12.5px; + line-height: 1.45; +} + +.confidence-wrap { + display: flex; + align-items: center; + gap: 8px; + margin-top: 6px; +} + +.confidence-bar { + flex: 1; + height: 5px; + border-radius: 3px; + background: rgba(71, 85, 105, 0.35); + overflow: hidden; +} + +.confidence-fill { + height: 100%; + border-radius: 3px; + background: linear-gradient(90deg, #0ea5e9, #38bdf8); +} + +.confidence-label { + font-family: var(--mono); + font-size: 10.5px; + color: var(--text-dim); +} + +.inference-method { + margin-top: 5px; + font-family: var(--mono); + font-size: 10px; + color: var(--text-faint); +} + +.contrary-evidence { + margin-top: 7px; + padding: 6px 8px; + border: 1px solid rgba(248, 113, 113, 0.4); + background: rgba(248, 113, 113, 0.07); + border-radius: 6px; +} + +.contrary-title { + font-size: 11px; + font-weight: 600; + color: var(--danger); +} + +.contrary-list { + margin: 4px 0 0; + padding-left: 16px; + font-family: var(--mono); + font-size: 10.5px; + color: var(--text-dim); +} + +/* deck tooltip (deck.gl injects its own element; keep it on-theme) */ + +.deck-tooltip { + font-family: var(--mono) !important; + font-size: 11px !important; + background: var(--bg-raised) !important; + color: var(--text) !important; + border: 1px solid var(--border); + border-radius: 6px; + padding: 4px 8px !important; +} diff --git a/intelligence/packages/globe/src/types.ts b/intelligence/packages/globe/src/types.ts new file mode 100644 index 0000000..db81820 --- /dev/null +++ b/intelligence/packages/globe/src/types.ts @@ -0,0 +1,234 @@ +// Local mirrors of the ontology shapes the globe consumes, plus the pinned +// /api/intel response contracts. Kept in lock-step with +// packages/ontology/src/types.ts — the JSON Schemas remain the source of truth. + +// --------------------------------------------------------------------------- +// Entities +// --------------------------------------------------------------------------- + +export type EntityType = + | "Grid" + | "GridRegion" + | "Operator" + | "Generator" + | "StorageAsset" + | "TransmissionAsset" + | "Interconnector" + | "Substation" + | "CoalBasin" + | "CoalMine" + | "GasBasin" + | "GasField" + | "GasProcessingPlant" + | "Pipeline" + | "HydroReservoir" + | "IndustrialFacility" + | "Smelter" + | "SteelMill" + | "Mine" + | "Refinery" + | "LNGPlant" + | "ChemicalPlant" + | "DataCentre" + | "Port" + | "Terminal" + | "Berth" + | "Railway" + | "Train" + | "Vessel" + | "Company" + | "Commodity" + | "Product" + | "Market" + | "Destination" + | "Sensor" + | "TelemetryFeed" + | "Observation" + | "Source" + | "Evidence" + | "Inference" + | "ObservabilityGap"; + +export type Observability = + | "KNOWN_LIVE" + | "ESTIMATED" + | "NOT_OBSERVABLE" + | "INSTRUMENTATION_OPPORTUNITY" + | "UNCLASSIFIED"; + +export interface GeoGeometry { + type: string; + coordinates: number[] | number[][] | number[][][] | number[][][][]; +} + +export interface EntityRecord { + entity_id: string; + entity_type: EntityType; + name: string; + observability: Observability; + geometry?: GeoGeometry | null; + country?: string | null; + grid_id?: string | null; + properties?: Record; + [key: string]: unknown; +} + +// --------------------------------------------------------------------------- +// Observability gaps +// --------------------------------------------------------------------------- + +export type ValueBand = "low" | "medium" | "high" | "critical"; +export type GapPriority = "P1" | "P2" | "P3" | "P4"; +export type GapStatus = "open" | "proxied" | "instrumented" | "wont_do"; +export type CostBand = "low" | "medium" | "high"; + +export interface InstrumentationOption { + kind: string; + notes?: string; + indicative_cost_band?: CostBand; +} + +export interface ObservabilityGapRecord { + gap_id: string; + entity_id: string; + desired_metric: string; + commercial_value: ValueBand; + strategic_value: ValueBand; + priority: GapPriority; + status: GapStatus; + current_source?: string | null; + best_available_proxy?: { + source_id: string; + metric: string; + derivation?: string; + } | null; + required_resolution?: string; + required_accuracy?: string; + instrumentation_options?: InstrumentationOption[]; + estimated_complexity?: CostBand; + notes?: string; +} + +// --------------------------------------------------------------------------- +// Inferences +// --------------------------------------------------------------------------- + +export type InferenceStatus = + | "INFERRED" + | "CORROBORATED" + | "CONTESTED" + | "VERIFIED" + | "RETRACTED"; + +export interface EvidenceRef { + kind: string; + ref: string; + role: "supports" | "contradicts"; +} + +export interface InferenceRecord { + inference_id: string; + claim: string; + tier: "A" | "B" | "C"; + confidence: number; + status: InferenceStatus; + method: string; + evidence?: EvidenceRef[]; + contrary_evidence?: EvidenceRef[]; + produced_at?: string; + [key: string]: unknown; +} + +// --------------------------------------------------------------------------- +// SDUI panel document +// --------------------------------------------------------------------------- + +export interface SduiPanel { + panel: string; + props?: Record; + data?: { endpoint: string }; +} + +export interface SduiPanelDoc { + entity_id: string; + entity_type: EntityType; + situation: string[]; + layout: SduiPanel[]; +} + +// --------------------------------------------------------------------------- +// /api/intel response contracts (pinned — the api package codes to the same) +// --------------------------------------------------------------------------- + +export interface GridInfo { + grid_id: string; + name: string; + kind: string; + countries: string[]; + frequency_hz: number; + live_data: { availability: string; notes?: string; source_ids?: string[] }; + parent_grid?: string | null; +} + +export interface GridsResponse { + grids: GridInfo[]; +} + +export interface GridSummaryResponse { + grid: GridInfo; + regions: Array<{ entity_id: string; name: string }>; + live: { + demand_mw: { value: number; event_time: string } | null; + price: { value: number; event_time: string } | null; + generation_mw: { value: number } | null; + flows: Array<{ + entity_id: string; + name: string; + mw: number; + event_time: string; + }>; + }; +} + +export interface EntitiesResponse { + entities: EntityRecord[]; +} + +export interface StateEntry { + metric: string; + value: number | string | Record; + unit?: string; + event_time: string; + quality?: string; + source_id?: string; +} + +export interface EntityStateResponse { + entity_id: string; + states: StateEntry[]; +} + +export interface GapsResponse { + gaps: ObservabilityGapRecord[]; +} + +/** `data:` payload of SSE events `state.updated` / `state.corrected`. */ +export interface StreamStateEvent { + state: { + entity_id: string; + metric: string; + value: number | string | Record; + unit?: string; + event_time: string; + }; + previous?: unknown; +} + +/** Structured value carried by `vessel.position` state. */ +export interface VesselPositionValue { + lat: number; + lon: number; + speed_kn?: number; + heading_deg?: number; + [key: string]: unknown; +} diff --git a/intelligence/packages/globe/src/vite-env.d.ts b/intelligence/packages/globe/src/vite-env.d.ts new file mode 100644 index 0000000..11f02fe --- /dev/null +++ b/intelligence/packages/globe/src/vite-env.d.ts @@ -0,0 +1 @@ +/// diff --git a/intelligence/packages/globe/vite.config.ts b/intelligence/packages/globe/vite.config.ts new file mode 100644 index 0000000..8d94ec4 --- /dev/null +++ b/intelligence/packages/globe/vite.config.ts @@ -0,0 +1,22 @@ +import { defineConfig, loadEnv } from "vite"; + +// Dev serves the app same-origin and proxies /api/intel to the api package +// (default 127.0.0.1:4200; override with VITE_API_TARGET). The SSE stream +// rides the same proxy. Build stays fully self-contained — no CDN, no fonts. +export default defineConfig(({ mode }) => { + const env = loadEnv(mode, process.cwd(), ""); + const target = env.VITE_API_TARGET || "http://127.0.0.1:4200"; + return { + server: { + proxy: { + "/api/intel": { + target, + changeOrigin: true, + }, + }, + }, + build: { + chunkSizeWarningLimit: 1500, + }, + }; +}); From 219272a9c0fe45441e0f048806409a0a9350f298 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 02:06:44 +0000 Subject: [PATCH 16/16] ci(intelligence): path-filtered workflow + end-to-end smoke Four jobs scoped to intelligence/** only (spec/CLI/MCP work never waits on them): build + 171 unit tests, registry/schema validation with referential integrity, the end-to-end smoke (fixture replay -> medallion -> state -> graph -> REST/SDUI -> SSE, 14 assertions), and the globe bundle build. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014yVGqBtvM1zNAd9tvUVNzm --- .github/workflows/intelligence.yml | 82 ++++++++++++++++++++++++++++++ intelligence/scripts/smoke.sh | 81 +++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 .github/workflows/intelligence.yml create mode 100644 intelligence/scripts/smoke.sh diff --git a/.github/workflows/intelligence.yml b/.github/workflows/intelligence.yml new file mode 100644 index 0000000..8a6aa0c --- /dev/null +++ b/.github/workflows/intelligence.yml @@ -0,0 +1,82 @@ +# CI for the intelligence/ extension workspace only. Path-filtered so spec/CLI/MCP +# work never waits on it (mirrors conformance.yml's approach). +name: intelligence + +on: + push: + branches: [main] + paths: + - "intelligence/**" + - ".github/workflows/intelligence.yml" + pull_request: + paths: + - "intelligence/**" + - ".github/workflows/intelligence.yml" + +jobs: + build-test: + name: Build + unit tests + runs-on: ubuntu-latest + defaults: + run: + working-directory: intelligence + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + cache-dependency-path: intelligence/package-lock.json + - run: npm ci + - run: npm run build + - run: npm test + + validate-data: + name: Validate registries + schemas + runs-on: ubuntu-latest + defaults: + run: + working-directory: intelligence + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + cache-dependency-path: intelligence/package-lock.json + - run: npm ci + - run: npm run build -w @pact-tailor/ontology && npm run build -w @pact-tailor/registry + - run: npm run validate:data + + smoke: + name: End-to-end smoke (fixtures → API → SSE) + runs-on: ubuntu-latest + defaults: + run: + working-directory: intelligence + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + cache-dependency-path: intelligence/package-lock.json + - run: npm ci + - run: npm run build + - run: bash scripts/smoke.sh + + globe-build: + name: Globe bundle builds + runs-on: ubuntu-latest + defaults: + run: + working-directory: intelligence + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + cache-dependency-path: intelligence/package-lock.json + - run: npm ci + - run: npm run build:globe diff --git a/intelligence/scripts/smoke.sh b/intelligence/scripts/smoke.sh new file mode 100644 index 0000000..0cb1e81 --- /dev/null +++ b/intelligence/scripts/smoke.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# End-to-end smoke: replay fixtures → bronze/silver → state → graph → API → SSE. +# Style follows tools/matter-smoke.sh: bash + curl + python3, numbered assertions. +# Usage: bash scripts/smoke.sh [port] (run from intelligence/, after npm run build) +set -u +PORT="${1:-4271}" +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +VAR="$(mktemp -d)" +API_LOG="$VAR/api.log" +PASS=0 +FAIL=0 +API_PID="" + +say() { printf '%s\n' "$*"; } +ok() { PASS=$((PASS + 1)); say " ok $1"; } +bad() { FAIL=$((FAIL + 1)); say " FAIL $1"; } +check() { # check + local desc="$1"; shift + if "$@" >/dev/null 2>&1; then ok "$desc"; else bad "$desc"; fi +} + +cleanup() { + [ -n "$API_PID" ] && kill "$API_PID" 2>/dev/null + rm -rf "$VAR" +} +trap cleanup EXIT + +say "== 1. replay ingestion (fixtures → bronze → silver)" +if node "$ROOT/packages/connectors/dist/cli.js" ingest --replay --var "$VAR" --data "$ROOT/data" >"$VAR/ingest.log" 2>&1; then + ok "pt-ingest --replay exits 0" +else + bad "pt-ingest --replay exits 0 (see $VAR/ingest.log)" + cat "$VAR/ingest.log" +fi +check "bronze artifacts written" bash -c "ls \"$VAR\"/bronze/*/*/*/*/*.bin" +check "silver observations written" bash -c "ls \"$VAR\"/silver/*/*.jsonl" + +say "== 2. API boot (replay mode)" +node "$ROOT/packages/api/dist/cli.js" --port "$PORT" --var "$VAR" --data "$ROOT/data" --replay >"$API_LOG" 2>&1 & +API_PID=$! +for _ in $(seq 1 50); do + curl -sf "http://127.0.0.1:$PORT/healthz" >/dev/null 2>&1 && break + sleep 0.2 +done +check "healthz responds" curl -sf "http://127.0.0.1:$PORT/healthz" + +api() { curl -sf "http://127.0.0.1:$PORT$1"; } + +say "== 3. REST surface" +check "meta reports entities > 100" bash -c "api() { curl -sf http://127.0.0.1:$PORT\$1; }; api /api/intel/meta | python3 -c 'import sys,json; m=json.load(sys.stdin); assert m[\"counts\"][\"entities\"] > 100'" +check "NEM grid summary has live demand" bash -c "curl -sf http://127.0.0.1:$PORT/api/intel/grids/grid:au-nem/summary | python3 -c 'import sys,json; s=json.load(sys.stdin); assert s[\"live\"][\"demand_mw\"] is not None'" +check "Gladstone PS entity + telemetry state" bash -c "curl -sf 'http://127.0.0.1:$PORT/api/intel/entities/gen:au-nem:gladstone-ps' | python3 -c 'import sys,json; e=json.load(sys.stdin); assert e[\"entity\"][\"name\"].startswith(\"Gladstone\"); assert isinstance(e[\"state\"], list)'" +check "bbox query finds Gladstone cluster" bash -c "curl -sf 'http://127.0.0.1:$PORT/api/intel/entities?bbox=150.5,-24.5,151.8,-23.3' | python3 -c 'import sys,json; r=json.load(sys.stdin); ids=[e[\"entity_id\"] for e in r[\"entities\"]]; assert \"smelter:au-qld:boyne-island\" in ids'" +check "graph traversal: what feeds Boyne" bash -c "curl -sf 'http://127.0.0.1:$PORT/api/intel/entities/smelter:au-qld:boyne-island/graph?direction=upstream&depth=2' | python3 -c 'import sys,json; g=json.load(sys.stdin); names=[n[\"entity_id\"] for n in g[\"nodes\"]]; assert \"gen:au-nem:gladstone-ps\" in names'" +check "gap registry served" bash -c "curl -sf 'http://127.0.0.1:$PORT/api/intel/gaps?entity=smelter:au-qld:boyne-island' | python3 -c 'import sys,json; g=json.load(sys.stdin); assert len(g[\"gaps\"]) >= 1'" +check "SDUI composes for Boyne (schema-valid, endpoint-only panels)" bash -c "curl -sf 'http://127.0.0.1:$PORT/api/intel/sdui/smelter:au-qld:boyne-island' | python3 -c ' +import sys, json +d = json.load(sys.stdin) +assert d[\"entity_id\"] == \"smelter:au-qld:boyne-island\" +assert len(d[\"layout\"]) >= 3 +for p in d[\"layout\"]: + data = p.get(\"data\") + if data is not None: + assert data[\"endpoint\"].startswith(\"/api/intel/\") +'" +check "inferences endpoint serves derived class only" bash -c "curl -sf 'http://127.0.0.1:$PORT/api/intel/inferences' | python3 -c 'import sys,json; r=json.load(sys.stdin); assert isinstance(r[\"inferences\"], list)'" +check "404 shape" bash -c "test \"\$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:$PORT/api/intel/entities/gen:au-nem:nope)\" = 404" + +say "== 4. SSE stream delivers at least one event" +SSE_OUT="$VAR/sse.txt" +curl -sN --max-time 12 "http://127.0.0.1:$PORT/api/intel/stream" >"$SSE_OUT" 2>/dev/null +if grep -qE "^event: (hello|state\.updated|state\.corrected|inference)" "$SSE_OUT" && grep -qE "^event: (state\.updated|state\.corrected|inference)" "$SSE_OUT"; then + ok "stream produced hello + at least one data event" +else + bad "stream produced hello + at least one data event" + head -20 "$SSE_OUT" +fi + +say "" +say "smoke: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ] || exit 1